Skip to content

Commit 233ef2e

Browse files
TheMostDiligentTheMostDiligent
committed
TestingSwapChain: improve test image difference artifact naming
Distinguish tolerated differences from failed comparisons using _DIFF_OK and _DIFF_FAIL suffixes.
1 parent c8057a7 commit 233ef2e

3 files changed

Lines changed: 45 additions & 42 deletions

File tree

‎Tests/DiligentCoreAPITest/src/TestingSwapChainBaseTest.cpp‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -49,22 +49,22 @@ void ClearRenderTargetReference(IRenderDevice* pDevice,
4949
namespace
5050
{
5151

52-
class ComparisonFailureImageGuard
52+
class DifferenceImageGuard
5353
{
5454
public:
55-
ComparisonFailureImageGuard() :
56-
m_FileName{GetTestImageComparisonFailureFileName()}
55+
explicit DifferenceImageGuard(bool ComparisonPassed) :
56+
m_FileName{GetTestImageDifferenceFileName(ComparisonPassed)}
5757
{}
5858

59-
~ComparisonFailureImageGuard()
59+
~DifferenceImageGuard()
6060
{
6161
if (!m_FileName.empty())
6262
FileSystem::DeleteFile(m_FileName.c_str());
6363
}
6464

6565
// clang-format off
66-
ComparisonFailureImageGuard(const ComparisonFailureImageGuard&) = delete;
67-
ComparisonFailureImageGuard& operator=(const ComparisonFailureImageGuard&) = delete;
66+
DifferenceImageGuard(const DifferenceImageGuard&) = delete;
67+
DifferenceImageGuard& operator=(const DifferenceImageGuard&) = delete;
6868
// clang-format on
6969

7070
const std::string& GetFileName() const
@@ -89,10 +89,10 @@ TEST(TestingSwapChainBaseTest, ExactComparisonAcceptsIdenticalImages)
8989
8,
9090
};
9191

92-
std::unordered_map<std::string, int> FailureCounters;
92+
std::unordered_map<std::string, int> DifferenceCounters;
9393
CompareTestImages(Pixels.data(), 8, Pixels.data(), 8, 2, 1,
94-
TEX_FORMAT_RGBA8_UNORM, FailureCounters);
95-
EXPECT_TRUE(FailureCounters.empty());
94+
TEX_FORMAT_RGBA8_UNORM, DifferenceCounters);
95+
EXPECT_TRUE(DifferenceCounters.empty());
9696
}
9797

9898
TEST(TestingSwapChainBaseTest, ToleratesConfiguredImageDifferences)
@@ -122,12 +122,13 @@ TEST(TestingSwapChainBaseTest, ToleratesConfiguredImageDifferences)
122122
ComparisonAttribs.MaxChannelError = 2;
123123
ComparisonAttribs.MaxBadPixelRatio = 0.5f;
124124

125-
std::unordered_map<std::string, int> FailureCounters;
126-
ComparisonFailureImageGuard FailureImageGuard;
125+
std::unordered_map<std::string, int> DifferenceCounters;
126+
DifferenceImageGuard ImageGuard{true};
127127
CompareTestImages(Reference.data(), 8, Actual.data(), 8, 2, 1,
128-
TEX_FORMAT_RGBA8_UNORM, FailureCounters, ComparisonAttribs);
129-
EXPECT_EQ(FailureCounters.size(), 1u);
130-
EXPECT_TRUE(FileSystem::FileExists(FailureImageGuard.GetFileName().c_str()));
128+
TEX_FORMAT_RGBA8_UNORM, DifferenceCounters, ComparisonAttribs);
129+
EXPECT_EQ(DifferenceCounters.size(), 1u);
130+
EXPECT_TRUE(FileSystem::FileExists(ImageGuard.GetFileName().c_str()));
131+
EXPECT_NE(ImageGuard.GetFileName().find("_DIFF_OK"), std::string::npos);
131132
}
132133

133134
#if !PLATFORM_WEB
@@ -159,31 +160,32 @@ TEST(TestingSwapChainBaseTest, ReportsOnlyNonEmptyDifferenceCategories)
159160
ComparisonAttribs.MaxChannelError = 2;
160161

161162
testing::internal::CaptureStdout();
162-
std::unordered_map<std::string, int> FailureCounters;
163-
ComparisonFailureImageGuard FailureImageGuard;
163+
std::unordered_map<std::string, int> DifferenceCounters;
164+
DifferenceImageGuard ImageGuard{true};
164165
CompareTestImages(Reference.data(), 8, Actual.data(), 8, 2, 1,
165-
TEX_FORMAT_RGBA8_UNORM, FailureCounters, ComparisonAttribs);
166+
TEX_FORMAT_RGBA8_UNORM, DifferenceCounters, ComparisonAttribs);
166167
const std::string Output = testing::internal::GetCapturedStdout();
167168

168169
EXPECT_NE(Output.find("1 of 2 pixels differ but remain within the per-channel error threshold 2; "
169170
"maximum channel error is 2"),
170171
std::string::npos);
171172
EXPECT_EQ(Output.find("exceed the threshold"), std::string::npos);
172-
EXPECT_TRUE(FileSystem::FileExists(FailureImageGuard.GetFileName().c_str()));
173+
EXPECT_TRUE(FileSystem::FileExists(ImageGuard.GetFileName().c_str()));
173174
}
174175

175-
TEST(TestingSwapChainBaseTest, AddsRenderDeviceTypeToFailureImageName)
176+
TEST(TestingSwapChainBaseTest, AddsRenderDeviceTypeToDifferenceImageName)
176177
{
177178
constexpr std::array<Uint8, 4> Reference{255, 255, 255, 255};
178179
constexpr std::array<Uint8, 4> Actual{0, 255, 255, 255};
179180

180-
std::unordered_map<std::string, int> FailureCounters;
181-
ComparisonFailureImageGuard FailureImageGuard;
181+
std::unordered_map<std::string, int> DifferenceCounters;
182+
DifferenceImageGuard ImageGuard{false};
182183
EXPECT_NONFATAL_FAILURE(
183184
CompareTestImages(Reference.data(), 4, Actual.data(), 4, 1, 1,
184-
TEX_FORMAT_RGBA8_UNORM, FailureCounters),
185+
TEX_FORMAT_RGBA8_UNORM, DifferenceCounters),
185186
"Image rendered by the test differs from the reference image");
186-
EXPECT_TRUE(FileSystem::FileExists(FailureImageGuard.GetFileName().c_str()));
187+
EXPECT_TRUE(FileSystem::FileExists(ImageGuard.GetFileName().c_str()));
188+
EXPECT_NE(ImageGuard.GetFileName().find("_DIFF_FAIL"), std::string::npos);
187189
}
188190

189191
TEST(TestingSwapChainBaseTest, ReportsToleratedAndBadPixelStatistics)
@@ -228,11 +230,11 @@ TEST(TestingSwapChainBaseTest, ReportsToleratedAndBadPixelStatistics)
228230
TestImageComparisonAttribs ComparisonAttribs;
229231
ComparisonAttribs.MaxChannelError = 2;
230232

231-
std::unordered_map<std::string, int> FailureCounters;
232-
ComparisonFailureImageGuard FailureImageGuard;
233+
std::unordered_map<std::string, int> DifferenceCounters;
234+
DifferenceImageGuard ImageGuard{false};
233235
EXPECT_NONFATAL_FAILURE(
234236
CompareTestImages(Reference.data(), 16, Actual.data(), 16, 4, 1,
235-
TEX_FORMAT_RGBA8_UNORM, FailureCounters, ComparisonAttribs),
237+
TEX_FORMAT_RGBA8_UNORM, DifferenceCounters, ComparisonAttribs),
236238
"1 of 4 pixels differ but remain within the per-channel error threshold 2; maximum channel error is 2\n"
237239
"1 of 4 pixels (25%) exceed the threshold; maximum channel error is 23; "
238240
"up to 0 bad pixels are allowed");
@@ -268,7 +270,7 @@ void TestSnapshotComparison(Int32 Channel)
268270

269271
if (Channel >= 0)
270272
{
271-
ComparisonFailureImageGuard FailureImageGuard;
273+
DifferenceImageGuard ImageGuard{false};
272274
EXPECT_NONFATAL_FAILURE(
273275
pTestingSwapChain->CompareWithSnapshot(nullptr),
274276
"Image rendered by the test differs from the reference image");
@@ -314,7 +316,7 @@ void TestImageComparison(Int32 Channel)
314316

315317
if (Channel >= 0)
316318
{
317-
ComparisonFailureImageGuard FailureImageGuard;
319+
DifferenceImageGuard ImageGuard{false};
318320
EXPECT_NONFATAL_FAILURE(
319321
pTestingSwapChain->CompareWithSnapshot(nullptr),
320322
"Image rendered by the test differs from the reference image");

‎Tests/GPUTestFramework/include/TestingSwapChainBase.hpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ void CompareTestImages(const Uint8* pReferencePixels,
6565
Uint32 Width,
6666
Uint32 Height,
6767
TEXTURE_FORMAT Format,
68-
std::unordered_map<std::string, int>& FailureCounters,
68+
std::unordered_map<std::string, int>& DifferenceCounters,
6969
const TestImageComparisonAttribs& ComparisonAttribs = {},
7070
bool CompareAlpha = true);
7171

@@ -75,8 +75,9 @@ bool LoadTestImage(const char* FilePath,
7575
Uint32& Width,
7676
Uint32& Height);
7777

78-
/// Returns the backend-qualified file name for a comparison failure image.
79-
std::string GetTestImageComparisonFailureFileName(Uint32 FailureIndex = 0);
78+
/// Returns the backend-qualified file name for a comparison difference image.
79+
std::string GetTestImageDifferenceFileName(bool ComparisonPassed,
80+
Uint32 DifferenceIndex = 0);
8081

8182
/// Writes the image to a PNG file. Alpha is omitted by default.
8283
void DumpTestImage(const Uint8* pPixels,
@@ -368,7 +369,7 @@ class TestingSwapChainBase : public RefCountedObject<SwapChainCombinedBaseInterf
368369

369370
m_pContext->MapTextureSubresource(m_pStagingTexture, 0, 0, MAP_READ, MapFlag, nullptr, MapData);
370371
CompareTestImages(m_ReferenceData.data(), m_ReferenceDataPitch, static_cast<const Uint8*>(MapData.pData), MapData.Stride,
371-
m_SwapChainDesc.Width, m_SwapChainDesc.Height, m_SwapChainDesc.ColorBufferFormat, m_FailureCounters,
372+
m_SwapChainDesc.Width, m_SwapChainDesc.Height, m_SwapChainDesc.ColorBufferFormat, m_DifferenceCounters,
372373
m_ImageComparisonAttribs, m_CompareAlpha);
373374

374375
m_pContext->UnmapTextureSubresource(m_pStagingTexture, 0, 0);
@@ -456,7 +457,7 @@ class TestingSwapChainBase : public RefCountedObject<SwapChainCombinedBaseInterf
456457
RefCntAutoPtr<ITextureView> m_pDSV;
457458
RefCntAutoPtr<ITexture> m_pStagingTexture;
458459

459-
std::unordered_map<std::string, int> m_FailureCounters;
460+
std::unordered_map<std::string, int> m_DifferenceCounters;
460461

461462
std::vector<Uint8> m_ReferenceData;
462463
Uint32 m_ReferenceDataPitch = 0;

‎Tests/GPUTestFramework/src/TestingSwapChainBase.cpp‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ bool LoadTestImage(const char* FilePath,
106106
return true;
107107
}
108108

109-
std::string GetTestImageComparisonFailureFileName(Uint32 FailureIndex)
109+
std::string GetTestImageDifferenceFileName(bool ComparisonPassed, Uint32 DifferenceIndex)
110110
{
111111
const auto* const TestInfo = ::testing::UnitTest::GetInstance()->current_test_info();
112112
GPUTestingEnvironment* const pEnvironment = GPUTestingEnvironment::GetInstance();
@@ -128,9 +128,9 @@ std::string GetTestImageComparisonFailureFileName(Uint32 FailureIndex)
128128
FileName += ValidateName(TestInfo->name());
129129
FileName += '_';
130130
FileName += GetRenderDeviceTypeShortString(pEnvironment->GetDevice()->GetDeviceInfo().Type);
131-
FileName += "_FAIL";
132-
if (FailureIndex > 0)
133-
FileName += std::to_string(FailureIndex);
131+
FileName += ComparisonPassed ? "_DIFF_OK" : "_DIFF_FAIL";
132+
if (DifferenceIndex > 0)
133+
FileName += std::to_string(DifferenceIndex);
134134
FileName += "_.png";
135135
return FileName;
136136
}
@@ -142,7 +142,7 @@ void CompareTestImages(const Uint8* pReferencePixels,
142142
Uint32 Width,
143143
Uint32 Height,
144144
TEXTURE_FORMAT Format,
145-
std::unordered_map<std::string, int>& FailureCounters,
145+
std::unordered_map<std::string, int>& DifferenceCounters,
146146
const TestImageComparisonAttribs& ComparisonAttribs,
147147
bool CompareAlpha)
148148
{
@@ -276,14 +276,14 @@ void CompareTestImages(const Uint8* pReferencePixels,
276276
}
277277
}
278278
}
279-
const std::string CounterKey = GetTestImageComparisonFailureFileName();
280-
auto& FailureCounter = FailureCounters[CounterKey];
281-
const std::string FileName = GetTestImageComparisonFailureFileName(static_cast<Uint32>(FailureCounter));
279+
const std::string CounterKey = GetTestImageDifferenceFileName(!ComparisonFailed);
280+
auto& DifferenceCounter = DifferenceCounters[CounterKey];
281+
const std::string FileName = GetTestImageDifferenceFileName(!ComparisonFailed, static_cast<Uint32>(DifferenceCounter));
282282
if (stbi_write_png(FileName.c_str(), Width * 2, Height * 2, 3, ReportImage.data(), (Width * 2) * 3) == 0)
283283
{
284284
LOG_ERROR_MESSAGE("Failed to write ", FileName);
285285
}
286-
++FailureCounter;
286+
++DifferenceCounter;
287287
}
288288

289289
if (!ComparisonFailed)

0 commit comments

Comments
 (0)