[Visual Regression Tests] Fix xUnit error for updates & improve test workflow

README.md

@@ -49,6 +49,7 @@ Baselines are maintained separately for each platform (`macOS`, `Windows`, `Linu
 - `dotnet test --filter "DisplayName~DevExpress"` - runs tests for all controls implemented in DevExpress
 - `dotnet test --filter "DisplayName~Button"` - runs tests for Button under each of the themes it's implemented in
 - `dotnet test --list-tests` - lists all test cases
+- 🆕 `scripts/test.sh [dotnet test args]` - runs `dotnet test` and prints one deduplicated visual-regression summary at the end. It also lets you use shorthand filter values (for example `--filter EditableCombo` instead of `--filter "DisplayName~EditableCombo"`).
 
 **Updating baseline screenshots** when changes are intentional:
 - **macOS/Linux:** `UPDATE_BASELINES=true dotnet test [filters]`

scripts/test.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+normalize_filter_expression() {
+  local expression="$1"
+
+  if [[ "$expression" == *"~"* ||
+        "$expression" == *"="* ||
+        "$expression" == *"!"* ||
+        "$expression" == *"<"* ||
+        "$expression" == *">"* ||
+        "$expression" == *"|"* ||
+        "$expression" == *"&"* ||
+        "$expression" == *"("* ||
+        "$expression" == *")"* ]]; then
+    printf '%s' "$expression"
+    return
+  fi
+
+  printf 'DisplayName~%s' "$expression"
+}
+
+dotnet_args=()
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --filter)
+      if [[ $# -lt 2 ]]; then
+        dotnet_args+=("$1")
+        shift
+        continue
+      fi
+      dotnet_args+=("--filter" "$(normalize_filter_expression "$2")")
+      shift 2
+      ;;
+    --filter=*)
+      dotnet_args+=("--filter=$(normalize_filter_expression "${1#--filter=}")")
+      shift
+      ;;
+    *)
+      dotnet_args+=("$1")
+      shift
+      ;;
+  esac
+done
+
+summary_file="$(mktemp -t visual-regression-summary.XXXXXX)"
+cleanup() {
+  rm -f "$summary_file"
+}
+trap cleanup EXIT
+
+dotnet test "${dotnet_args[@]}" 2>&1 | while IFS= read -r line; do
+  normalized="$line"
+
+  if [[ "$normalized" =~ ^\[xUnit\.net[[:space:]][^]]+\][[:space:]]*(.*)$ ]]; then
+    normalized="${BASH_REMATCH[1]}"
+  fi
+
+  normalized="${normalized#"${normalized%%[![:space:]]*}"}"
+
+  if [[ "$normalized" =~ ^Visual\ regression\ detected\ for\ \[([^]]+)\]\ (.*)\ -\ ([^.]+)\.\ Diff\ saved\ to\ (.*)$ ]]; then
+    row="$(printf '%s\t%s\t%s\t%s\t%s' "Visual regression" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}")"
+    grep -Fxq "$row" "$summary_file" || printf '%s\n' "$row" >> "$summary_file"
+    continue
+  fi
+
+  if [[ "$normalized" =~ ^No\ baseline\ found\ for\ \[([^]]+)\]\ (.*)\ -\ ([^.]+)\.\ Saved\ screenshot\ to\ (.*)$ ]]; then
+    row="$(printf '%s\t%s\t%s\t%s\t%s' "No baseline" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}")"
+    grep -Fxq "$row" "$summary_file" || printf '%s\n' "$row" >> "$summary_file"
+    continue
+  fi
+
+  printf '%s\n' "$line"
+done
+
+dotnet_exit_code=${PIPESTATUS[0]}
+
+if [[ -s "$summary_file" ]]; then
+  printf '\n%s\n' "________________________________________________________________________________"
+  printf '\033[33;1m%s\033[0m\n' "Visual regression summary"
+  printf '\033[33;1m%-18s %-14s %-34s %-10s %s\033[0m\n' "Status" "Theme" "Page" "Variant" "Path"
+  while IFS=$'\t' read -r status theme page variant path; do
+    printf '\033[33;1m%-18s %-14s %-34s %-10s %s\033[0m\n' "$status" "[$theme]" "$page" "$variant" "$path"
+  done < "$summary_file"
+  printf '%s\n\n' "________________________________________________________________________________"
+fi
+
+exit "$dotnet_exit_code"

tests/Devolutions.AvaloniaControls.VisualTests/VisualRegressionTests.cs

@@ -279,10 +279,10 @@ public void TestPage(Type pageType, string themeName, Type? viewModelType)
       Dispatcher.UIThread.RunJobs();
 
       // 4. Test Light Mode
-      CaptureAndCompare("", ThemeVariant.Light);
+      CaptureAndCompare(window, pageName, themeName, "", ThemeVariant.Light);
 
       // 5. Test Dark Mode
-      CaptureAndCompare("_dark", ThemeVariant.Dark);
+      CaptureAndCompare(window, pageName, themeName, "_dark", ThemeVariant.Dark);
     }
     finally
     {
@@ -291,52 +291,53 @@ public void TestPage(Type pageType, string themeName, Type? viewModelType)
       window.Content = null;
       Dispatcher.UIThread.RunJobs();
     }
+  }
 
-
-    void CaptureAndCompare(string suffix, ThemeVariant variant)
+  [System.Diagnostics.StackTraceHidden]
+  private static void CaptureAndCompare(Window window, string pageName, string themeName, string suffix, ThemeVariant variant)
+  {
+    if (Application.Current != null)
     {
-      if (Application.Current != null)
-      {
-        Application.Current.RequestedThemeVariant = variant;
-      }
+      Application.Current.RequestedThemeVariant = variant;
+    }
 
-      // Wait for layout and theme application
-      Dispatcher.UIThread.RunJobs();
+    // Wait for layout and theme application
+    Dispatcher.UIThread.RunJobs();
 
-      // Capture
-      WriteableBitmap? frame = window.CaptureRenderedFrame();
-      if (frame == null) throw new Exception($"Failed to capture frame for {variant}");
-      using WriteableBitmap bitmap = frame;
+    // Capture
+    WriteableBitmap? frame = window.CaptureRenderedFrame();
+    if (frame == null) throw new Exception($"Failed to capture frame for {variant}");
+    using WriteableBitmap bitmap = frame;
 
-      // Save and Compare
-      var fileName = $"{pageName}{suffix}.png";
-      string baselinePath = Path.Combine(BaselinesDirectory, themeName, fileName);
-      string testPath = Path.Combine(TestResultsDirectory, themeName, fileName);
-      string diffPath = Path.Combine(TestDiffsDirectory, themeName, $"{pageName}{suffix}_diff.png");
+    // Save and Compare
+    var fileName = $"{pageName}{suffix}.png";
+    string baselinePath = Path.Combine(BaselinesDirectory, themeName, fileName);
+    string testPath = Path.Combine(TestResultsDirectory, themeName, fileName);
+    string diffPath = Path.Combine(TestDiffsDirectory, themeName, $"{pageName}{suffix}_diff.png");
 
-      // Ensure subdirectories exist
-      Directory.CreateDirectory(Path.GetDirectoryName(baselinePath)!);
-      Directory.CreateDirectory(Path.GetDirectoryName(testPath)!);
+    // Ensure subdirectories exist
+    Directory.CreateDirectory(Path.GetDirectoryName(baselinePath)!);
+    Directory.CreateDirectory(Path.GetDirectoryName(testPath)!);
 
-      bitmap.Save(testPath);
+    bitmap.Save(testPath);
 
-      if (Environment.GetEnvironmentVariable("UPDATE_BASELINES") == "true")
-      {
-        File.Copy(testPath, baselinePath, true);
-      }
+    if (Environment.GetEnvironmentVariable("UPDATE_BASELINES") == "true")
+    {
+      File.Copy(testPath, baselinePath, true);
+    }
 
-      if (File.Exists(baselinePath))
-      {
-        bool result = ImageComparer.CompareImages(baselinePath, testPath, diffPath);
-        Assert.True(result,
-          $"\n\u001b[1mVisual regression detected\u001b[0m for \u001b[1m[{themeName}] {pageName} - {variant}\u001b[0m. Diff saved to {Path.GetDirectoryName(diffPath)}");
-      }
-      else
+    if (File.Exists(baselinePath))
+    {
+      bool passed = ImageComparer.CompareImages(baselinePath, testPath, diffPath);
+      if (!passed)
       {
-        Assert.Fail(
-          $"\n\u001b[1mNo baseline found\u001b[0m for \u001b[1m[{themeName}] {pageName} - {variant}\u001b[0m. Saved screenshot to {testPath}");
+        Assert.Fail($"Visual regression detected for [{themeName}] {pageName} - {variant}. Diff saved to {Path.GetDirectoryName(diffPath)}");
       }
     }
+    else
+    {
+      Assert.Fail($"No baseline found for [{themeName}] {pageName} - {variant}. Saved screenshot to {testPath}");
+    }
   }
 }
 
@@ -349,18 +350,69 @@ internal static void Run()
 
     if (Environment.GetEnvironmentVariable("UPDATE_BASELINES") == "true")
     {
-      Console.ForegroundColor = ConsoleColor.Yellow;
-      Console.WriteLine("\n\n" + new string('!', 80));
-      Console.WriteLine("\u001b[1mWARNING: UPDATE_BASELINES environment variable is set to 'true'!\u001b[0m");
-      Console.WriteLine("Visual regression baselines will be updated.");
-      Console.WriteLine("If this was not intentional:");
-      Console.WriteLine("");
-      Console.WriteLine(" 🚨 \u001b[1mYou may abort with Ctrl+C.\u001b[0m  🚨 ");
-      Console.WriteLine("");
-      Console.WriteLine("Remove the variable from the current shell with:");
-      Console.WriteLine("`export UPDATE_BASELINES=false`");
-      Console.WriteLine(new string('!', 80) + "\n");
-      Console.ResetColor();
+      // xUnit v3 spawns the test process multiple times (discovery + execution).
+      // Use a sentinel file to show the warning banner only once per 30-second window.
+      string sentinelFile = Path.Combine(Path.GetTempPath(), "avalonia-update-baselines-warning.tmp");
+      bool shouldShowBanner = ShouldShowUpdateBaselinesBanner(sentinelFile);
+
+      if (shouldShowBanner)
+      {
+        TryMarkUpdateBaselinesBannerShown(sentinelFile);
+        TextWriter stderr = Console.Error;
+        stderr.WriteLine("\n\n" + new string('_', 80));
+        stderr.WriteLine("\u001b[33m\u001b[1mWARNING: UPDATE_BASELINES environment variable is set to 'true'!\u001b[0m");
+        stderr.WriteLine("Visual regression baselines will be updated.");
+        stderr.WriteLine("If this was not intentional:");
+        stderr.WriteLine("");
+        stderr.WriteLine(" 🚨 \u001b[1mYou may abort with Ctrl+C.\u001b[0m  🚨 ");
+        stderr.WriteLine("");
+        stderr.WriteLine("Remove the variable from the current shell with:");
+        stderr.WriteLine("`export UPDATE_BASELINES=false`");
+        stderr.WriteLine(new string('_', 80) + "\n");
+      }
+    }
+  }
+
+  private static bool ShouldShowUpdateBaselinesBanner(string sentinelFile)
+  {
+    try
+    {
+      if (File.Exists(sentinelFile))
+      {
+        var age = DateTime.UtcNow - File.GetLastWriteTimeUtc(sentinelFile);
+        return age.TotalSeconds > 30;
+      }
+
+      return true;
+    }
+    catch (IOException)
+    {
+      return true;
+    }
+    catch (UnauthorizedAccessException)
+    {
+      return true;
+    }
+    catch (NotSupportedException)
+    {
+      return true;
+    }
+  }
+
+  private static void TryMarkUpdateBaselinesBannerShown(string sentinelFile)
+  {
+    try
+    {
+      File.WriteAllText(sentinelFile, "");
+    }
+    catch (IOException)
+    {
+    }
+    catch (UnauthorizedAccessException)
+    {
+    }
+    catch (NotSupportedException)
+    {
     }
   }