From ea292ee3e379a913a37d64bd156f546d65af3368 Mon Sep 17 00:00:00 2001 From: Anna Malchow-Perryman Date: Thu, 18 Jun 2026 14:07:33 -0400 Subject: [PATCH 1/4] Fix xUnit error for baseline updates --- .../VisualRegressionTests.cs | 112 ++++++++++-------- 1 file changed, 64 insertions(+), 48 deletions(-) diff --git a/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs b/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs index e032f389..ae76262b 100644 --- a/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs +++ b/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs @@ -279,10 +279,10 @@ public void TestPage(Type pageType, string themeName, Type? viewModelType) Dispatcher.UIThread.RunJobs(); // 4. Test Light Mode - CaptureAndCompare("", ThemeVariant.Light); + CaptureAndCompare(window, pageName, themeName, "", ThemeVariant.Light); // 5. Test Dark Mode - CaptureAndCompare("_dark", ThemeVariant.Dark); + CaptureAndCompare(window, pageName, themeName, "_dark", ThemeVariant.Dark); } finally { @@ -291,52 +291,55 @@ public void TestPage(Type pageType, string themeName, Type? viewModelType) window.Content = null; Dispatcher.UIThread.RunJobs(); } + } - - void CaptureAndCompare(string suffix, ThemeVariant variant) + [System.Diagnostics.StackTraceHidden] + private static void CaptureAndCompare(Window window, string pageName, string themeName, string suffix, ThemeVariant variant) + { + if (Application.Current != null) { - if (Application.Current != null) - { - Application.Current.RequestedThemeVariant = variant; - } + Application.Current.RequestedThemeVariant = variant; + } - // Wait for layout and theme application - Dispatcher.UIThread.RunJobs(); + // Wait for layout and theme application + Dispatcher.UIThread.RunJobs(); - // Capture - WriteableBitmap? frame = window.CaptureRenderedFrame(); - if (frame == null) throw new Exception($"Failed to capture frame for {variant}"); - using WriteableBitmap bitmap = frame; + // Capture + WriteableBitmap? frame = window.CaptureRenderedFrame(); + if (frame == null) throw new Exception($"Failed to capture frame for {variant}"); + using WriteableBitmap bitmap = frame; - // Save and Compare - var fileName = $"{pageName}{suffix}.png"; - string baselinePath = Path.Combine(BaselinesDirectory, themeName, fileName); - string testPath = Path.Combine(TestResultsDirectory, themeName, fileName); - string diffPath = Path.Combine(TestDiffsDirectory, themeName, $"{pageName}{suffix}_diff.png"); + // Save and Compare + var fileName = $"{pageName}{suffix}.png"; + string baselinePath = Path.Combine(BaselinesDirectory, themeName, fileName); + string testPath = Path.Combine(TestResultsDirectory, themeName, fileName); + string diffPath = Path.Combine(TestDiffsDirectory, themeName, $"{pageName}{suffix}_diff.png"); - // Ensure subdirectories exist - Directory.CreateDirectory(Path.GetDirectoryName(baselinePath)!); - Directory.CreateDirectory(Path.GetDirectoryName(testPath)!); + // Ensure subdirectories exist + Directory.CreateDirectory(Path.GetDirectoryName(baselinePath)!); + Directory.CreateDirectory(Path.GetDirectoryName(testPath)!); - bitmap.Save(testPath); + bitmap.Save(testPath); - if (Environment.GetEnvironmentVariable("UPDATE_BASELINES") == "true") - { - File.Copy(testPath, baselinePath, true); - } + if (Environment.GetEnvironmentVariable("UPDATE_BASELINES") == "true") + { + File.Copy(testPath, baselinePath, true); + } - if (File.Exists(baselinePath)) - { - bool result = ImageComparer.CompareImages(baselinePath, testPath, diffPath); - Assert.True(result, - $"\n\u001b[1mVisual regression detected\u001b[0m for \u001b[1m[{themeName}] {pageName} - {variant}\u001b[0m. Diff saved to {Path.GetDirectoryName(diffPath)}"); - } - else + if (File.Exists(baselinePath)) + { + bool passed = ImageComparer.CompareImages(baselinePath, testPath, diffPath); + if (!passed) { - Assert.Fail( - $"\n\u001b[1mNo baseline found\u001b[0m for \u001b[1m[{themeName}] {pageName} - {variant}\u001b[0m. Saved screenshot to {testPath}"); + Console.Error.WriteLine($"\u001b[33;1mVisual regression detected\u001b[0m for \u001b[33;1m[{themeName}] {pageName} - {variant}\u001b[0m. Diff saved to {Path.GetDirectoryName(diffPath)}"); + Assert.Fail($"Visual regression detected for [{themeName}] {pageName} - {variant}. Diff saved to {Path.GetDirectoryName(diffPath)}"); } } + else + { + Console.Error.WriteLine($"\u001b[33;1mNo baseline found\u001b[0m for \u001b[33;1m[{themeName}] {pageName} - {variant}\u001b[0m. Saved screenshot to {testPath}"); + Assert.Fail($"No baseline found for [{themeName}] {pageName} - {variant}. Saved screenshot to {testPath}"); + } } } @@ -349,18 +352,31 @@ internal static void Run() if (Environment.GetEnvironmentVariable("UPDATE_BASELINES") == "true") { - Console.ForegroundColor = ConsoleColor.Yellow; - Console.WriteLine("\n\n" + new string('!', 80)); - Console.WriteLine("\u001b[1mWARNING: UPDATE_BASELINES environment variable is set to 'true'!\u001b[0m"); - Console.WriteLine("Visual regression baselines will be updated."); - Console.WriteLine("If this was not intentional:"); - Console.WriteLine(""); - Console.WriteLine(" 🚨 \u001b[1mYou may abort with Ctrl+C.\u001b[0m 🚨 "); - Console.WriteLine(""); - Console.WriteLine("Remove the variable from the current shell with:"); - Console.WriteLine("`export UPDATE_BASELINES=false`"); - Console.WriteLine(new string('!', 80) + "\n"); - Console.ResetColor(); + // xUnit v3 spawns the test process multiple times (discovery + execution). + // Use a sentinel file to show the warning banner only once per 30-second window. + string sentinelFile = Path.Combine(Path.GetTempPath(), "avalonia-update-baselines-warning.tmp"); + bool shouldShowBanner = true; + if (File.Exists(sentinelFile)) + { + var age = DateTime.UtcNow - File.GetLastWriteTimeUtc(sentinelFile); + shouldShowBanner = age.TotalSeconds > 30; + } + + if (shouldShowBanner) + { + File.WriteAllText(sentinelFile, ""); + TextWriter stderr = Console.Error; + stderr.WriteLine("\n\n" + new string('_', 80)); + stderr.WriteLine("\u001b[33m\u001b[1mWARNING: UPDATE_BASELINES environment variable is set to 'true'!\u001b[0m"); + stderr.WriteLine("Visual regression baselines will be updated."); + stderr.WriteLine("If this was not intentional:"); + stderr.WriteLine(""); + stderr.WriteLine(" 🚨 \u001b[1mYou may abort with Ctrl+C.\u001b[0m 🚨 "); + stderr.WriteLine(""); + stderr.WriteLine("Remove the variable from the current shell with:"); + stderr.WriteLine("`export UPDATE_BASELINES=false`"); + stderr.WriteLine(new string('_', 80) + "\n"); + } } } From e4d641b933f995bdb70ba7153b29d4c3027f459f Mon Sep 17 00:00:00 2001 From: Anna Malchow-Perryman Date: Thu, 18 Jun 2026 14:48:42 -0400 Subject: [PATCH 2/4] Create test script with nicer output --- README.md | 1 + scripts/test.sh | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 scripts/test.sh diff --git a/README.md b/README.md index c4e70f53..1412e66d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ Baselines are maintained separately for each platform (`macOS`, `Windows`, `Linu - `dotnet test --filter "DisplayName~DevExpress"` - runs tests for all controls implemented in DevExpress - `dotnet test --filter "DisplayName~Button"` - runs tests for Button under each of the themes it's implemented in - `dotnet test --list-tests` - lists all test cases +- `scripts/test.sh [dotnet test args]` - runs `dotnet test` and prints one deduplicated visual-regression summary at the end **Updating baseline screenshots** when changes are intentional: - **macOS/Linux:** `UPDATE_BASELINES=true dotnet test [filters]` diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 00000000..8a7863d8 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -uo pipefail + +summary_file="$(mktemp -t visual-regression-summary.XXXXXX)" +cleanup() { + rm -f "$summary_file" +} +trap cleanup EXIT + +dotnet test "$@" 2>&1 | while IFS= read -r line; do + normalized="$line" + + if [[ "$normalized" =~ ^\[xUnit\.net[[:space:]][^]]+\][[:space:]]*(.*)$ ]]; then + normalized="${BASH_REMATCH[1]}" + fi + + normalized="${normalized#"${normalized%%[![:space:]]*}"}" + + if [[ "$normalized" =~ ^Visual\ regression\ detected\ for\ \[([^]]+)\]\ (.*)\ -\ ([^.]+)\.\ Diff\ saved\ to\ (.*)$ ]]; then + row="$(printf '%s\t%s\t%s\t%s\t%s' "Visual regression" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}")" + grep -Fxq "$row" "$summary_file" || printf '%s\n' "$row" >> "$summary_file" + continue + fi + + if [[ "$normalized" =~ ^No\ baseline\ found\ for\ \[([^]]+)\]\ (.*)\ -\ ([^.]+)\.\ Saved\ screenshot\ to\ (.*)$ ]]; then + row="$(printf '%s\t%s\t%s\t%s\t%s' "No baseline" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}")" + grep -Fxq "$row" "$summary_file" || printf '%s\n' "$row" >> "$summary_file" + continue + fi + + printf '%s\n' "$line" +done + +dotnet_exit_code=${PIPESTATUS[0]} + +if [[ -s "$summary_file" ]]; then + printf '\n%s\n' "________________________________________________________________________________" + printf '\033[33;1m%s\033[0m\n' "Visual regression summary" + printf '\033[33;1m%-18s %-14s %-34s %-10s %s\033[0m\n' "Status" "Theme" "Page" "Variant" "Path" + while IFS=$'\t' read -r status theme page variant path; do + printf '\033[33;1m%-18s %-14s %-34s %-10s %s\033[0m\n' "$status" "[$theme]" "$page" "$variant" "$path" + done < "$summary_file" + printf '%s\n\n' "________________________________________________________________________________" +fi + +exit "$dotnet_exit_code" From 9c62bf49e0b636b89a3d3dc69d4ca5ee6a723eb7 Mon Sep 17 00:00:00 2001 From: Anna Malchow-Perryman Date: Thu, 18 Jun 2026 15:01:47 -0400 Subject: [PATCH 3/4] Streamline filter syntax when using the script --- README.md | 2 +- scripts/test.sh | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1412e66d..0c7e6fec 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Baselines are maintained separately for each platform (`macOS`, `Windows`, `Linu - `dotnet test --filter "DisplayName~DevExpress"` - runs tests for all controls implemented in DevExpress - `dotnet test --filter "DisplayName~Button"` - runs tests for Button under each of the themes it's implemented in - `dotnet test --list-tests` - lists all test cases -- `scripts/test.sh [dotnet test args]` - runs `dotnet test` and prints one deduplicated visual-regression summary at the end +- 🆕 `scripts/test.sh [dotnet test args]` - runs `dotnet test` and prints one deduplicated visual-regression summary at the end. It also lets you use shorthand filter values (for example `--filter EditableCombo` instead of `--filter "DisplayName~EditableCombo"`). **Updating baseline screenshots** when changes are intentional: - **macOS/Linux:** `UPDATE_BASELINES=true dotnet test [filters]` diff --git a/scripts/test.sh b/scripts/test.sh index 8a7863d8..20382119 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,13 +1,55 @@ #!/usr/bin/env bash set -uo pipefail +normalize_filter_expression() { + local expression="$1" + + if [[ "$expression" == *"~"* || + "$expression" == *"="* || + "$expression" == *"!"* || + "$expression" == *"<"* || + "$expression" == *">"* || + "$expression" == *"|"* || + "$expression" == *"&"* || + "$expression" == *"("* || + "$expression" == *")"* ]]; then + printf '%s' "$expression" + return + fi + + printf 'DisplayName~%s' "$expression" +} + +dotnet_args=() +while [[ $# -gt 0 ]]; do + case "$1" in + --filter) + if [[ $# -lt 2 ]]; then + dotnet_args+=("$1") + shift + continue + fi + dotnet_args+=("--filter" "$(normalize_filter_expression "$2")") + shift 2 + ;; + --filter=*) + dotnet_args+=("--filter=$(normalize_filter_expression "${1#--filter=}")") + shift + ;; + *) + dotnet_args+=("$1") + shift + ;; + esac +done + summary_file="$(mktemp -t visual-regression-summary.XXXXXX)" cleanup() { rm -f "$summary_file" } trap cleanup EXIT -dotnet test "$@" 2>&1 | while IFS= read -r line; do +dotnet test "${dotnet_args[@]}" 2>&1 | while IFS= read -r line; do normalized="$line" if [[ "$normalized" =~ ^\[xUnit\.net[[:space:]][^]]+\][[:space:]]*(.*)$ ]]; then From 38cd2691c30de0d1586f4d8a4a7de43bcfe2dee6 Mon Sep 17 00:00:00 2001 From: Anna Malchow-Perryman Date: Thu, 18 Jun 2026 15:50:44 -0400 Subject: [PATCH 4/4] Address reviewer feedback on visual test output Make baseline-update banner sentinel file handling best-effort so module initialization cannot fail when temp file I/O is unavailable. Also remove duplicate stderr emission from visual test failures and rely on Assert.Fail messages, which keeps both raw dotnet output and scripts/test.sh summary output less noisy. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../VisualRegressionTests.cs | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs b/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs index ae76262b..6e16816b 100644 --- a/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs +++ b/tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs @@ -331,13 +331,11 @@ private static void CaptureAndCompare(Window window, string pageName, string the bool passed = ImageComparer.CompareImages(baselinePath, testPath, diffPath); if (!passed) { - Console.Error.WriteLine($"\u001b[33;1mVisual regression detected\u001b[0m for \u001b[33;1m[{themeName}] {pageName} - {variant}\u001b[0m. Diff saved to {Path.GetDirectoryName(diffPath)}"); Assert.Fail($"Visual regression detected for [{themeName}] {pageName} - {variant}. Diff saved to {Path.GetDirectoryName(diffPath)}"); } } else { - Console.Error.WriteLine($"\u001b[33;1mNo baseline found\u001b[0m for \u001b[33;1m[{themeName}] {pageName} - {variant}\u001b[0m. Saved screenshot to {testPath}"); Assert.Fail($"No baseline found for [{themeName}] {pageName} - {variant}. Saved screenshot to {testPath}"); } } @@ -355,16 +353,11 @@ internal static void Run() // xUnit v3 spawns the test process multiple times (discovery + execution). // Use a sentinel file to show the warning banner only once per 30-second window. string sentinelFile = Path.Combine(Path.GetTempPath(), "avalonia-update-baselines-warning.tmp"); - bool shouldShowBanner = true; - if (File.Exists(sentinelFile)) - { - var age = DateTime.UtcNow - File.GetLastWriteTimeUtc(sentinelFile); - shouldShowBanner = age.TotalSeconds > 30; - } + bool shouldShowBanner = ShouldShowUpdateBaselinesBanner(sentinelFile); if (shouldShowBanner) { - File.WriteAllText(sentinelFile, ""); + TryMarkUpdateBaselinesBannerShown(sentinelFile); TextWriter stderr = Console.Error; stderr.WriteLine("\n\n" + new string('_', 80)); stderr.WriteLine("\u001b[33m\u001b[1mWARNING: UPDATE_BASELINES environment variable is set to 'true'!\u001b[0m"); @@ -380,6 +373,49 @@ internal static void Run() } } + private static bool ShouldShowUpdateBaselinesBanner(string sentinelFile) + { + try + { + if (File.Exists(sentinelFile)) + { + var age = DateTime.UtcNow - File.GetLastWriteTimeUtc(sentinelFile); + return age.TotalSeconds > 30; + } + + return true; + } + catch (IOException) + { + return true; + } + catch (UnauthorizedAccessException) + { + return true; + } + catch (NotSupportedException) + { + return true; + } + } + + private static void TryMarkUpdateBaselinesBannerShown(string sentinelFile) + { + try + { + File.WriteAllText(sentinelFile, ""); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + catch (NotSupportedException) + { + } + } + private static void EnsureAvaloniaLicenseKeyIsLoaded() { if (!string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("AVALONIA_LICENSE_KEY")))