Improve test image comparison diagnostics

Report tolerated and failing pixel differences separately.
Include backend names in comparison failure images.
Keep exact image comparisons silent.

Author: TheMostDiligent

Tests/DiligentCoreAPITest/src/TestingSwapChainBaseTest.cpp

@@ -96,13 +96,9 @@ TEST(TestingSwapChainBaseTest, ToleratesConfiguredImageDifferences)
 class ComparisonFailureImageGuard
 {
 public:
-    ComparisonFailureImageGuard()
-    {
-        const testing::TestInfo* const TestInfo = testing::UnitTest::GetInstance()->current_test_info();
-        VERIFY_EXPR(TestInfo != nullptr);
-        if (TestInfo != nullptr)
-            m_FileName = std::string{TestInfo->test_suite_name()} + "." + TestInfo->name() + "_FAIL_.png";
-    }
+    ComparisonFailureImageGuard() :
+        m_FileName{GetTestImageComparisonFailureFileName()}
+    {}
 
     ~ComparisonFailureImageGuard()
     {
@@ -115,10 +111,116 @@ class ComparisonFailureImageGuard
     ComparisonFailureImageGuard& operator=(const ComparisonFailureImageGuard&) = delete;
     // clang-format on
 
+    const std::string& GetFileName() const
+    {
+        return m_FileName;
+    }
+
 private:
     std::string m_FileName;
 };
 
+TEST(TestingSwapChainBaseTest, ReportsOnlyNonEmptyDifferenceCategories)
+{
+    constexpr std::array<Uint8, 8> Reference{
+        10,
+        20,
+        30,
+        255,
+        40,
+        50,
+        60,
+        255,
+    };
+    constexpr std::array<Uint8, 8> Actual{
+        12,
+        20,
+        30,
+        255, // Maximum error 2: tolerated.
+        40,
+        50,
+        60,
+        255,
+    };
+
+    TestImageComparisonAttribs ComparisonAttribs;
+    ComparisonAttribs.MaxChannelError = 2;
+
+    testing::internal::CaptureStdout();
+    std::unordered_map<std::string, int> FailureCounters;
+    CompareTestImages(Reference.data(), 8, Actual.data(), 8, 2, 1,
+                      TEX_FORMAT_RGBA8_UNORM, FailureCounters, ComparisonAttribs);
+    const std::string Output = testing::internal::GetCapturedStdout();
+
+    EXPECT_NE(Output.find("1 of 2 pixels differ but remain within the per-channel error threshold 2"), std::string::npos);
+    EXPECT_EQ(Output.find("exceed the threshold"), std::string::npos);
+}
+
+TEST(TestingSwapChainBaseTest, AddsRenderDeviceTypeToFailureImageName)
+{
+    constexpr std::array<Uint8, 4> Reference{255, 255, 255, 255};
+    constexpr std::array<Uint8, 4> Actual{0, 255, 255, 255};
+
+    std::unordered_map<std::string, int> FailureCounters;
+    ComparisonFailureImageGuard          FailureImageGuard;
+    EXPECT_NONFATAL_FAILURE(
+        CompareTestImages(Reference.data(), 4, Actual.data(), 4, 1, 1,
+                          TEX_FORMAT_RGBA8_UNORM, FailureCounters),
+        "Image rendered by the test differs from the reference image");
+    EXPECT_TRUE(FileSystem::FileExists(FailureImageGuard.GetFileName().c_str()));
+}
+
+TEST(TestingSwapChainBaseTest, ReportsToleratedAndBadPixelStatistics)
+{
+    constexpr std::array<Uint8, 16> Reference{
+        10,
+        20,
+        30,
+        255,
+        40,
+        50,
+        60,
+        255,
+        70,
+        80,
+        90,
+        255,
+        100,
+        110,
+        120,
+        255,
+    };
+    constexpr std::array<Uint8, 16> Actual{
+        12,
+        20,
+        30,
+        255, // Maximum error 2: tolerated.
+        40,
+        27,
+        60,
+        255, // Maximum error 23: bad.
+        70,
+        80,
+        90,
+        255,
+        100,
+        110,
+        120,
+        255,
+    };
+
+    TestImageComparisonAttribs ComparisonAttribs;
+    ComparisonAttribs.MaxChannelError = 2;
+
+    std::unordered_map<std::string, int> FailureCounters;
+    ComparisonFailureImageGuard          FailureImageGuard;
+    EXPECT_NONFATAL_FAILURE(
+        CompareTestImages(Reference.data(), 16, Actual.data(), 16, 4, 1,
+                          TEX_FORMAT_RGBA8_UNORM, FailureCounters, ComparisonAttribs),
+        "1 of 4 pixels differ but remain within the per-channel error threshold 2; maximum channel error is 2\n"
+        "1 of 4 pixels (25%) exceed the threshold; maximum channel error is 23");
+}
+
 void TestSnapshotComparisonFailure(Uint32 Channel)
 {
     ASSERT_LT(Channel, 4u);

Tests/GPUTestFramework/include/TestingSwapChainBase.hpp

@@ -27,6 +27,7 @@
 #pragma once
 
 #include <cstring>
+#include <string>
 #include <vector>
 #include <unordered_map>
 
@@ -74,6 +75,9 @@ bool LoadTestImage(const char*         FilePath,
                    Uint32&             Width,
                    Uint32&             Height);
 
+/// Returns the backend-qualified file name for a comparison failure image.
+std::string GetTestImageComparisonFailureFileName(Uint32 FailureIndex = 0);
+
 /// Writes the image to a PNG file. Alpha is omitted by default.
 void DumpTestImage(const Uint8*   pPixels,
                    Uint64         PixelsStride,

Tests/GPUTestFramework/src/TestingSwapChainBase.cpp

@@ -30,6 +30,7 @@
 #include <algorithm>
 #include <limits>
 #include <memory>
+#include <sstream>
 
 #include "TestingSwapChainBase.hpp"
 #include "GraphicsAccessories.hpp"
@@ -105,6 +106,35 @@ bool LoadTestImage(const char*         FilePath,
     return true;
 }
 
+std::string GetTestImageComparisonFailureFileName(Uint32 FailureIndex)
+{
+    const auto* const            TestInfo     = ::testing::UnitTest::GetInstance()->current_test_info();
+    GPUTestingEnvironment* const pEnvironment = GPUTestingEnvironment::GetInstance();
+    VERIFY_EXPR(TestInfo != nullptr);
+    VERIFY_EXPR(pEnvironment != nullptr && pEnvironment->GetDevice() != nullptr);
+
+    const auto ValidateName = [](const std::string& src) {
+        std::string dst = src;
+        for (char& c : dst)
+        {
+            if (c == '.' || c == '\\' || c == '/')
+                c = '_';
+        }
+        return dst;
+    };
+
+    std::string FileName{ValidateName(TestInfo->test_suite_name())};
+    FileName += '.';
+    FileName += ValidateName(TestInfo->name());
+    FileName += '_';
+    FileName += GetRenderDeviceTypeShortString(pEnvironment->GetDevice()->GetDeviceInfo().Type);
+    FileName += "_FAIL";
+    if (FailureIndex > 0)
+        FileName += std::to_string(FailureIndex);
+    FileName += "_.png";
+    return FileName;
+}
+
 void CompareTestImages(const Uint8*                          pReferencePixels,
                        Uint64                                RefPixelsStride,
                        const Uint8*                          pPixels,
@@ -161,29 +191,64 @@ void CompareTestImages(const Uint8*                          pReferencePixels,
     if (bIsIdentical)
         return;
 
-    Uint64 BadPixelCount         = 0;
-    Uint32 MaxObservedChannelErr = 0;
+    Uint64 ToleratedPixelCount      = 0;
+    Uint64 BadPixelCount            = 0;
+    Uint32 MaxToleratedChannelError = 0;
+    Uint32 MaxBadChannelError       = 0;
     for (Uint32 Row = 0; Row < Height; ++Row)
     {
         for (Uint32 Col = 0; Col < Width; ++Col)
         {
-            bool BadPixel = false;
+            Uint32 PixelMaxChannelError = 0;
             for (Uint32 Component = 0; Component < ComponentCount; ++Component)
             {
                 const Uint32 RefValue = pReferencePixels[Row * RefPixelsStride + Col * 4 + Component];
                 const Uint32 Value    = pPixels[Row * PixelsStride + Col * 4 + Component];
                 const Uint32 Error    = RefValue > Value ? RefValue - Value : Value - RefValue;
-                MaxObservedChannelErr = std::max(MaxObservedChannelErr, Error);
-                BadPixel |= Error > ComparisonAttribs.MaxChannelError;
+                PixelMaxChannelError  = std::max(PixelMaxChannelError, Error);
+            }
+
+            if (PixelMaxChannelError > ComparisonAttribs.MaxChannelError)
+            {
+                ++BadPixelCount;
+                MaxBadChannelError = std::max(MaxBadChannelError, PixelMaxChannelError);
+            }
+            else if (PixelMaxChannelError > 0)
+            {
+                ++ToleratedPixelCount;
+                MaxToleratedChannelError = std::max(MaxToleratedChannelError, PixelMaxChannelError);
             }
-            BadPixelCount += BadPixel ? 1 : 0;
         }
     }
 
+    if (ToleratedPixelCount == 0 && BadPixelCount == 0)
+        return;
+
     const Uint64 PixelCount = Uint64{Width} * Height;
+
+    std::ostringstream Statistics;
+    if (ToleratedPixelCount > 0)
+    {
+        Statistics << ToleratedPixelCount << " of " << PixelCount
+                   << " pixels differ but remain within the per-channel error threshold "
+                   << Uint32{ComparisonAttribs.MaxChannelError} << "; maximum channel error is "
+                   << MaxToleratedChannelError;
+    }
+    if (BadPixelCount > 0)
+    {
+        if (ToleratedPixelCount > 0)
+            Statistics << '\n';
+
+        const double BadPixelPercentage = static_cast<double>(BadPixelCount) / static_cast<double>(PixelCount) * 100.0;
+        Statistics << BadPixelCount << " of " << PixelCount << " pixels (" << BadPixelPercentage
+                   << "%) exceed the threshold; maximum channel error is " << MaxBadChannelError;
+    }
+
     if (static_cast<double>(BadPixelCount) <=
         static_cast<double>(PixelCount) * ComparisonAttribs.MaxBadPixelRatio)
     {
+        LOG_WARNING_MESSAGE("Image rendered by the test differs from the reference image, but is within the configured tolerance:\n",
+                            Statistics.str());
         return;
     }
 
@@ -210,34 +275,15 @@ void CompareTestImages(const Uint8*                          pReferencePixels,
                 }
             }
         }
-        const auto* const TestInfo = ::testing::UnitTest::GetInstance()->current_test_info();
-
-        const auto ValidateName = [](const std::string& src) {
-            std::string dst = src;
-            for (char& c : dst)
-            {
-                if (c == '.' || c == '\\' || c == '/')
-                    c = '_';
-            }
-            return dst;
-        };
-
-        std::string FileName{ValidateName(TestInfo->test_suite_name())};
-        FileName += '.';
-        FileName += ValidateName(TestInfo->name());
-        auto& FailureCounter = FailureCounters[FileName];
-        FileName += "_FAIL";
-        if (FailureCounter > 0)
-            FileName += std::to_string(FailureCounter);
-        FileName += "_.png";
+        const std::string CounterKey     = GetTestImageComparisonFailureFileName();
+        auto&             FailureCounter = FailureCounters[CounterKey];
+        const std::string FileName       = GetTestImageComparisonFailureFileName(static_cast<Uint32>(FailureCounter));
         if (stbi_write_png(FileName.c_str(), Width * 2, Height * 2, 3, ReportImage.data(), (Width * 2) * 3) == 0)
         {
             LOG_ERROR_MESSAGE("Failed to write ", FileName);
         }
-        ADD_FAILURE() << "Image rendered by the test differs from the reference image: "
-                      << BadPixelCount << " of " << PixelCount << " pixels exceed the per-channel error threshold "
-                      << Uint32{ComparisonAttribs.MaxChannelError} << "; maximum observed channel error is "
-                      << MaxObservedChannelErr;
+        ADD_FAILURE() << "Image rendered by the test differs from the reference image:\n"
+                      << Statistics.str();
         ++FailureCounter;
     }
 }