From d37105bf8440a6475f9d98a34cf6dbb443a8f25e Mon Sep 17 00:00:00 2001 From: leileizhang Date: Fri, 18 Jul 2025 10:32:09 +0800 Subject: [PATCH] [UI Tests] Replace pixel-by-pixel image comparison with perceptual hash (pHash) for improved visual similarity detection (#40653) ## Summary of the Pull Request This PR replaces the previous pixel-by-pixel image comparison logic with a perceptual hash (pHash)-based comparison using the CoenM.ImageSharp.ImageHash library. **Removes the need for golden images from CI pipelines** Since the comparison is perceptual rather than binary, we no longer need to fetch pixel-perfect golden images from pipelines for validation. Developers can now capture screenshots locally and still get meaningful, robust comparisons. ### Why pHash? Unlike direct pixel comparison (which fails on minor rendering differences), pHash focuses on the overall structure and visual perception of the image. This provides several benefits: - Robust to minor differences: tolerates compression artifacts, anti-aliasing, subtle rendering changes, and border padding. - Resilient to resolution or format changes: works even if images are scaled or compressed differently. - Closer to human perception: more accurately reflects whether two images "look" the same to a person. ## PR Checklist - [ ] **Closes:** #xxx - [ ] **Communication:** I've discussed this with core contributors already. If the work hasn't been agreed, this work might be rejected - [ ] **Tests:** Added/updated and all pass - [ ] **Localization:** All end-user-facing strings can be localized - [ ] **Dev docs:** Added/updated - [ ] **New binaries:** Added on the required places - [ ] [JSON for signing](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ESRPSigning_core.json) for new binaries - [ ] [WXS for installer](https://github.com/microsoft/PowerToys/blob/main/installer/PowerToysSetup/Product.wxs) for new binaries and localization folder - [ ] [YML for CI pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/ci/templates/build-powertoys-steps.yml) for new test projects - [ ] [YML for signed pipeline](https://github.com/microsoft/PowerToys/blob/main/.pipelines/release.yml) - [ ] **Documentation updated:** If checked, please file a pull request on [our docs repo](https://github.com/MicrosoftDocs/windows-uwp/tree/docs/hub/powertoys) and link it here: #xxx ## Detailed Description of the Pull Request / Additional comments ## Validation Steps Performed --- .github/actions/spell-check/expect.txt | 1 + Directory.Packages.props | 1 + NOTICE.md | 1 + .../UITestAutomation/UITestAutomation.csproj | 1 + src/common/UITestAutomation/VisualAssert.cs | 73 +++++++++++++++---- 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/actions/spell-check/expect.txt b/.github/actions/spell-check/expect.txt index 3d22a06313..cd46ef497d 100644 --- a/.github/actions/spell-check/expect.txt +++ b/.github/actions/spell-check/expect.txt @@ -218,6 +218,7 @@ coclass CODENAME codereview Codespaces +Coen COINIT colid colorconv diff --git a/Directory.Packages.props b/Directory.Packages.props index f29669d9e7..23575616fe 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,6 +9,7 @@ + diff --git a/NOTICE.md b/NOTICE.md index b2232e4984..4dcc82579d 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1496,6 +1496,7 @@ SOFTWARE. - AdaptiveCards.Templating 2.0.5 - Appium.WebDriver 4.4.5 - Azure.AI.OpenAI 1.0.0-beta.17 +- CoenM.ImageSharp.ImageHash 1.3.6 - CommunityToolkit.Common 8.4.0 - CommunityToolkit.Labs.WinUI.Controls.MarkdownTextBlock 0.1.250703-build.2173 - CommunityToolkit.Mvvm 8.4.0 diff --git a/src/common/UITestAutomation/UITestAutomation.csproj b/src/common/UITestAutomation/UITestAutomation.csproj index fc9da3b983..17841e0a60 100644 --- a/src/common/UITestAutomation/UITestAutomation.csproj +++ b/src/common/UITestAutomation/UITestAutomation.csproj @@ -20,6 +20,7 @@ + diff --git a/src/common/UITestAutomation/VisualAssert.cs b/src/common/UITestAutomation/VisualAssert.cs index 692090440a..844db5b027 100644 --- a/src/common/UITestAutomation/VisualAssert.cs +++ b/src/common/UITestAutomation/VisualAssert.cs @@ -6,7 +6,11 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Drawing; using System.IO; +using CoenM.ImageHash; +using CoenM.ImageHash.HashAlgorithms; using Microsoft.VisualStudio.TestTools.UnitTesting; +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; namespace Microsoft.PowerToys.UITest { @@ -127,34 +131,75 @@ namespace Microsoft.PowerToys.UITest } /// - /// Test if two images are equal bit-by-bit + /// Test if two images are equal using ImageHash comparison /// /// baseline image /// test image /// true if are equal,otherwise false private static bool AreEqual(Bitmap baselineImage, Bitmap testImage) { - if (baselineImage.Width != testImage.Width || baselineImage.Height != testImage.Height) + try { - return false; + // Define a threshold for similarity percentage + const int SimilarityThreshold = 95; + + // Use CoenM.ImageHash for perceptual hash comparison + var hashAlgorithm = new AverageHash(); + + // Convert System.Drawing.Bitmap to SixLabors.ImageSharp.Image + using var baselineImageSharp = ConvertBitmapToImageSharp(baselineImage); + using var testImageSharp = ConvertBitmapToImageSharp(testImage); + + // Calculate hashes for both images + var baselineHash = hashAlgorithm.Hash(baselineImageSharp); + var testHash = hashAlgorithm.Hash(testImageSharp); + + // Compare hashes using CompareHash method + // Returns similarity percentage (0-100, where 100 is identical) + var similarity = CompareHash.Similarity(baselineHash, testHash); + + // Consider images equal if similarity is very high + // Allow for minor rendering differences (threshold can be adjusted) + return similarity >= SimilarityThreshold; // 95% similarity threshold } - - // WinAppDriver sometimes adds a border to the screenshot (around 2 pix width), and it is not always consistent. - // So we exclude the border when comparing the images, and usually it is the edge of the windows, won't affect the comparison. - int excludeBorderWidth = 5, excludeBorderHeight = 5; - - for (int x = excludeBorderWidth; x < baselineImage.Width - excludeBorderWidth; x++) + catch { - for (int y = excludeBorderHeight; y < baselineImage.Height - excludeBorderHeight; y++) + // Fallback to pixel-by-pixel comparison if hash comparison fails + if (baselineImage.Width != testImage.Width || baselineImage.Height != testImage.Height) { - if (!VisualHelper.PixIsSame(baselineImage.GetPixel(x, y), testImage.GetPixel(x, y))) + return false; + } + + // WinAppDriver sometimes adds a border to the screenshot (around 2 pix width), and it is not always consistent. + // So we exclude the border when comparing the images, and usually it is the edge of the windows, won't affect the comparison. + int excludeBorderWidth = 5, excludeBorderHeight = 5; + + for (int x = excludeBorderWidth; x < baselineImage.Width - excludeBorderWidth; x++) + { + for (int y = excludeBorderHeight; y < baselineImage.Height - excludeBorderHeight; y++) { - return false; + if (!VisualHelper.PixIsSame(baselineImage.GetPixel(x, y), testImage.GetPixel(x, y))) + { + return false; + } } } - } - return true; + return true; + } + } + + /// + /// Convert System.Drawing.Bitmap to SixLabors.ImageSharp.Image + /// + /// The bitmap to convert + /// ImageSharp Image + private static Image ConvertBitmapToImageSharp(Bitmap bitmap) + { + using var memoryStream = new MemoryStream(); + bitmap.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Png); + memoryStream.Position = 0; + return SixLabors.ImageSharp.Image.Load(memoryStream); } } }