From f816d555d9c85f8ffd9d627b56ccbe35d1d9596c Mon Sep 17 00:00:00 2001
From: Shawn Isenhart <isen0011@umn.edu>
Date: Wed, 8 May 2024 14:19:43 -0400
Subject: [PATCH] Allow marking tests Pending when they fail audits

Works on #21.

This doesn't completely fix the problem that is identified in #21, but
it is a step in that direction.  The idea is that, if the configuration
setting `accessibility_audit_skip_on_error` is set to true, if an `axe`
violation is detected, instead of generating an error it will instead
mark the test as `skipped` with the `axe` output as the message in the
skip output.

There is a problem with this approach, which is the same problem
identified in issue #7 - as soon as a single test is marked skipped or
failed, then the rest of the test is skipped.  This means that if a
test has both failing application logic and failing accessibility
validation, the first of those tests to fail will be the one that is
reported.  For a system test, since any interaction with the page will
trigger the accessibility tests, it is likely that the test will be
marked as skipped and the application logic failure will be hidden.

This can be solved by running the tests a second time with
`accessibility_audit_enabled = false` and comparing the results, but
that does require two runs through the test suite.
---
 README.md                                          | 11 +++++++++++
 .../audit_system_test_extensions.rb                |  9 ++++++++-
 spec/system/audit_assertions_spec.rb               | 14 ++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c2b06b2..6e6b3d8 100644
--- a/README.md
+++ b/README.md
@@ -141,6 +141,17 @@ class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
 end
 ```
 
+How can I make all of the failing tests just be pending instead of failures?
+---
+
+You can allow automated auditing, but mark tests that fail the auditing be marked as `pending` instead of `failed`:
+
+```ruby
+class ApplicationSystemTestCase < ActionDispatch::SystemTestCase
+  self.accessibility_audit_skip_on_error = true
+end
+```
+
 How can I turn off auditing for a block of code?
 ---
 
diff --git a/lib/capybara_accessibility_audit/audit_system_test_extensions.rb b/lib/capybara_accessibility_audit/audit_system_test_extensions.rb
index 9624e59..ab8b984 100644
--- a/lib/capybara_accessibility_audit/audit_system_test_extensions.rb
+++ b/lib/capybara_accessibility_audit/audit_system_test_extensions.rb
@@ -14,6 +14,7 @@ module AuditSystemTestExtensions
     included do
       class_attribute :accessibility_audit_after_methods, default: Set.new
       class_attribute :accessibility_audit_enabled, default: true
+      class_attribute :accessibility_audit_skip_on_error, default: false
       class_attribute :accessibility_audit_options, default: ActiveSupport::OrderedOptions.new
 
       MODAL_METHODS.each do |method|
@@ -51,6 +52,7 @@ def skip_accessibility_audit_after(*methods)
 
     def with_accessibility_audits(**options, &block)
       accessibility_audit_enabled = self.accessibility_audit_enabled
+      accessibility_audit_skip_on_error = self.accessibility_audit_skip_on_error
       self.accessibility_audit_enabled = true
 
       if options.present?
@@ -103,7 +105,12 @@ def assert_no_accessibility_violations(**options)
       axe_matcher = Axe::Matchers::BeAxeClean.new
       axe_matcher = options.inject(axe_matcher) { |matcher, option| matcher.public_send(*option) }
 
-      assert axe_matcher.matches?(page), axe_matcher.failure_message
+      page_axe_match = axe_matcher.matches?(page)
+      if !page_axe_match && self.accessibility_audit_skip_on_error
+        skip(axe_matcher.failure_message)
+      else
+        assert page_axe_match, axe_matcher.failure_message
+      end
     end
   end
 end
diff --git a/spec/system/audit_assertions_spec.rb b/spec/system/audit_assertions_spec.rb
index 60021f9..26fe4cf 100644
--- a/spec/system/audit_assertions_spec.rb
+++ b/spec/system/audit_assertions_spec.rb
@@ -63,6 +63,20 @@
     end
   end
 
+  describe "Make failing tests Pending" do
+    before do
+      self.accessibility_audit_skip_on_error = true
+    end
+
+    it "marks violation test as pending" do
+      allow_any_instance_of(RSpec::Core::Pending).to receive(:skip)
+      visit violations_path
+      click_on "Violate rule: label"
+      go_back
+      click_on "Violate rule: image-alt"
+    end
+  end
+
   describe "Disabling" do
     before do
       self.accessibility_audit_enabled = false