From feaf1246fc2a5876991d5a4011dbfd7d5ca9c0b6 Mon Sep 17 00:00:00 2001 From: "tbhb.dev Agent" Date: Sat, 14 Nov 2026 09:28:28 -0500 Subject: [PATCH 1/3] fix(quality): keep the gate verdict when the summary is unwritable A GITHUB_STEP_SUMMARY that cannot be written no longer ends the gate under set -e before the verdict and survivor listing; it warns on stderr. The gate test rule now inherits GITHUB_STEP_SUMMARY, which Bazel's test sandbox otherwise drops, and the stale timeout figures leave the cap comment. Generated-by: LLM effort=medium;harness=claude-code:2.1.318;model=anthropic:claude-sonnet-5-5;session=sha-256-64:f1227f050617f845;surface=sdk-cli --- quality/private/mutation_gate.bzl | 14 ++++--- quality/private/mutation_gate.sh.tpl | 7 +++- quality/tests/BUILD.bazel | 26 ++++++++++++- quality/tests/gate_output_test.bzl | 56 ++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/quality/private/mutation_gate.bzl b/quality/private/mutation_gate.bzl index 32d2de3..31208e2 100644 --- a/quality/private/mutation_gate.bzl +++ b/quality/private/mutation_gate.bzl @@ -11,10 +11,9 @@ visibility("//quality/...") # Most timed-out mutants, in percent of total, that still count as killed. A # starved CPU makes tests time out, so past this share timeouts say the run -# was starved, not that the tests caught the mutants. One real run on 2 CPUs -# timed out 212 of 849 mutants, about 25%. The gate and both runners read this -# one value: the gate through its template, the runners through the -# environment variable below, which each requires. +# was starved, not that the tests caught the mutants. The gate and both +# runners read this one value: the gate through its template, the runners +# through the environment variable below, which each requires. MUTATION_MAX_TIMEOUT_PERCENT = 10 MUTATION_RUNNER_ENV = {"MUTATION_MAX_TIMEOUT_PERCENT": str(MUTATION_MAX_TIMEOUT_PERCENT)} @@ -51,7 +50,12 @@ def _mutation_gate_impl(ctx): is_executable = True, ) executable, runfiles = shell_executable(ctx, script, ctx.runfiles(files = reports)) - return [DefaultInfo(executable = executable, runfiles = runfiles)] + return [ + DefaultInfo(executable = executable, runfiles = runfiles), + # Bazel's test sandbox passes only listed variables; the gate appends + # its timeout line to the job summary. + RunEnvironmentInfo(inherited_environment = ["GITHUB_STEP_SUMMARY"]), + ] mutation_gate_test = rule( implementation = _mutation_gate_impl, diff --git a/quality/private/mutation_gate.sh.tpl b/quality/private/mutation_gate.sh.tpl index 1aeae87..dac62bf 100644 --- a/quality/private/mutation_gate.sh.tpl +++ b/quality/private/mutation_gate.sh.tpl @@ -24,7 +24,8 @@ # # A timed-out mutant counts as killed and stays listed (its mutant.timeout or # mutant.timed_out lines print). The count line goes to stdout, and to -# $GITHUB_STEP_SUMMARY when that is set. +# $GITHUB_STEP_SUMMARY when that is set; a failed write there only warns on +# stderr. # # Fails on: any lived, not_covered or errored mutant; timeouts above # MAX_TIMEOUT_PERCENT of total, in the sum and in any one report; an @@ -199,7 +200,9 @@ if ((timeout > 0 && total > 0)); then # mutmut says timeout, gomutants timed_out; the lines print as reported. printf '%s\n' "$mutants" | grep -E '^mutant\.(timeout|timed_out)=' || true if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then - echo "$timeout_line" >>"$GITHUB_STEP_SUMMARY" + # An unwritable summary must not end the run before the verdict. + echo "$timeout_line" >>"$GITHUB_STEP_SUMMARY" || + echo "mutation_gate: could not write to GITHUB_STEP_SUMMARY" >&2 fi fi diff --git a/quality/tests/BUILD.bazel b/quality/tests/BUILD.bazel index b200791..cd91757 100644 --- a/quality/tests/BUILD.bazel +++ b/quality/tests/BUILD.bazel @@ -6,7 +6,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("//quality:defs.bzl", "mutation_gate_test") load("//repo:defs.bzl", "repo_files") load("//testing:defs.bzl", "expect_failure_test") -load(":gate_output_test.bzl", "gate_output_test") +load(":gate_output_test.bzl", "gate_output_test", "gate_unwritable_summary_test") load(":mutation_changes_test.bzl", "mutation_changes_test") load(":slice_test.bzl", "changes_slice") @@ -123,6 +123,30 @@ gate_output_test( summary_lines = ["mutation_gate: 10 timed out of 100 (10.0%), counted as killed"], ) +# An unwritable job summary (here a missing directory) ends neither a passing +# nor a failing run early: the verdict, the stdout line and the listed +# timeouts stay, and stderr notes the failed write. +gate_unwritable_summary_test( + name = "unwritable_summary_passes_test", + gate = ":timeout_below_cap_gate", + lines = [ + "mutation_gate: 5 timed out of 100 (5.0%), counted as killed", + "mutant.timeout=pkg.x_f__mutmut_5", + "mutation_gate: ok: every mutant was killed", + ], + status = 0, +) + +gate_unwritable_summary_test( + name = "unwritable_summary_fails_test", + gate = ":proof_timeout_with_survivor", + lines = [ + "mutation_gate: 5 timed out of 100 (5.0%), counted as killed", + "mutant.timeout=pkg.x_f__mutmut_5", + ], + status = 1, +) + # Timed-out mutants are listed in the gate's output, passing or failing. expect_failure_test( name = "proof_timeout_with_survivor_lists_timeouts_fails", diff --git a/quality/tests/gate_output_test.bzl b/quality/tests/gate_output_test.bzl index 900e343..5517d00 100644 --- a/quality/tests/gate_output_test.bzl +++ b/quality/tests/gate_output_test.bzl @@ -73,3 +73,59 @@ each of `summary_lines`, then once more with it unset.""", "summary_lines": attr.string_list(doc = "Whole lines the summary file must hold."), }, ) + +_UNWRITABLE_SCRIPT = """#!/usr/bin/env bash +set -uo pipefail +gate="$PWD/{gate}" +work="$TEST_TMPDIR/gate_unwritable" +mkdir -p "$work" +status=0 +GITHUB_STEP_SUMMARY="$work/missing/summary.md" "$gate" >"$work/out.log" 2>"$work/err.log" || status=$? +if ((status != {status})); then + echo "gate_unwritable_summary_test: FAIL: the gate exited $status, not {status}" >&2 + cat "$work/out.log" "$work/err.log" >&2 + exit 1 +fi +for line in {lines}; do + if ! grep -qxF -- "$line" "$work/out.log"; then + echo "gate_unwritable_summary_test: FAIL: stdout lacks '$line'" >&2 + cat "$work/out.log" >&2 + exit 1 + fi +done +if ! grep -qxF -- "mutation_gate: could not write to GITHUB_STEP_SUMMARY" "$work/err.log"; then + echo "gate_unwritable_summary_test: FAIL: stderr lacks the could-not-write note" >&2 + cat "$work/err.log" >&2 + exit 1 +fi +""" + +def _unwritable_impl(ctx): + gate = ctx.attr.gate[DefaultInfo].files_to_run.executable + script = ctx.actions.declare_file(ctx.label.name + ".sh") + ctx.actions.write( + output = script, + content = _UNWRITABLE_SCRIPT.format( + gate = gate.short_path, + status = ctx.attr.status, + lines = " ".join([shell.quote(line) for line in ctx.attr.lines]), + ), + is_executable = True, + ) + runfiles = ctx.runfiles(files = [gate]).merge(ctx.attr.gate[DefaultInfo].default_runfiles) + executable, runfiles = shell_executable(ctx, script, runfiles) + return [DefaultInfo(executable = executable, runfiles = runfiles)] + +gate_unwritable_summary_test = rule( + implementation = _unwritable_impl, + test = True, + toolchains = SHELL_TOOLCHAINS, + doc = """Passes when `gate`, run with `GITHUB_STEP_SUMMARY` in a missing directory, +still exits with `status`, prints each of `lines` to stdout and notes the +failed write on stderr.""", + attrs = SHELL_ATTRS | { + "gate": attr.label(mandatory = True, executable = True, cfg = "target"), + "lines": attr.string_list(mandatory = True, doc = "Whole lines the gate's stdout must hold."), + "status": attr.int(mandatory = True, values = [0, 1], doc = "The exit status the gate must keep."), + }, +) From da4217ee8503e3fe094f3ae91ee4e99654dbe987 Mon Sep 17 00:00:00 2001 From: "tbhb.dev Agent" Date: Sat, 14 Nov 2026 09:31:06 -0500 Subject: [PATCH 2/3] test(quality): rename the summary write tests to pass the spell check Generated-by: LLM effort=medium;harness=claude-code:2.1.318;model=anthropic:claude-sonnet-5-5;session=sha-256-64:f1227f050617f845;surface=sdk-cli --- quality/private/mutation_gate.sh.tpl | 2 +- quality/tests/BUILD.bazel | 12 ++++++------ quality/tests/gate_output_test.bzl | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/quality/private/mutation_gate.sh.tpl b/quality/private/mutation_gate.sh.tpl index dac62bf..9da9345 100644 --- a/quality/private/mutation_gate.sh.tpl +++ b/quality/private/mutation_gate.sh.tpl @@ -200,7 +200,7 @@ if ((timeout > 0 && total > 0)); then # mutmut says timeout, gomutants timed_out; the lines print as reported. printf '%s\n' "$mutants" | grep -E '^mutant\.(timeout|timed_out)=' || true if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then - # An unwritable summary must not end the run before the verdict. + # A summary that cannot be written must not end the run before the verdict. echo "$timeout_line" >>"$GITHUB_STEP_SUMMARY" || echo "mutation_gate: could not write to GITHUB_STEP_SUMMARY" >&2 fi diff --git a/quality/tests/BUILD.bazel b/quality/tests/BUILD.bazel index cd91757..a2e43cd 100644 --- a/quality/tests/BUILD.bazel +++ b/quality/tests/BUILD.bazel @@ -6,7 +6,7 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("//quality:defs.bzl", "mutation_gate_test") load("//repo:defs.bzl", "repo_files") load("//testing:defs.bzl", "expect_failure_test") -load(":gate_output_test.bzl", "gate_output_test", "gate_unwritable_summary_test") +load(":gate_output_test.bzl", "gate_output_test", "gate_summary_error_test") load(":mutation_changes_test.bzl", "mutation_changes_test") load(":slice_test.bzl", "changes_slice") @@ -123,11 +123,11 @@ gate_output_test( summary_lines = ["mutation_gate: 10 timed out of 100 (10.0%), counted as killed"], ) -# An unwritable job summary (here a missing directory) ends neither a passing +# A job summary that cannot be written (here a missing directory) ends neither a passing # nor a failing run early: the verdict, the stdout line and the listed # timeouts stay, and stderr notes the failed write. -gate_unwritable_summary_test( - name = "unwritable_summary_passes_test", +gate_summary_error_test( + name = "summary_error_passes_test", gate = ":timeout_below_cap_gate", lines = [ "mutation_gate: 5 timed out of 100 (5.0%), counted as killed", @@ -137,8 +137,8 @@ gate_unwritable_summary_test( status = 0, ) -gate_unwritable_summary_test( - name = "unwritable_summary_fails_test", +gate_summary_error_test( + name = "summary_error_fails_test", gate = ":proof_timeout_with_survivor", lines = [ "mutation_gate: 5 timed out of 100 (5.0%), counted as killed", diff --git a/quality/tests/gate_output_test.bzl b/quality/tests/gate_output_test.bzl index 5517d00..f0d4028 100644 --- a/quality/tests/gate_output_test.bzl +++ b/quality/tests/gate_output_test.bzl @@ -74,38 +74,38 @@ each of `summary_lines`, then once more with it unset.""", }, ) -_UNWRITABLE_SCRIPT = """#!/usr/bin/env bash +_SUMMARY_ERROR_SCRIPT = """#!/usr/bin/env bash set -uo pipefail gate="$PWD/{gate}" -work="$TEST_TMPDIR/gate_unwritable" +work="$TEST_TMPDIR/gate_summary_error" mkdir -p "$work" status=0 GITHUB_STEP_SUMMARY="$work/missing/summary.md" "$gate" >"$work/out.log" 2>"$work/err.log" || status=$? if ((status != {status})); then - echo "gate_unwritable_summary_test: FAIL: the gate exited $status, not {status}" >&2 + echo "gate_summary_error_test: FAIL: the gate exited $status, not {status}" >&2 cat "$work/out.log" "$work/err.log" >&2 exit 1 fi for line in {lines}; do if ! grep -qxF -- "$line" "$work/out.log"; then - echo "gate_unwritable_summary_test: FAIL: stdout lacks '$line'" >&2 + echo "gate_summary_error_test: FAIL: stdout lacks '$line'" >&2 cat "$work/out.log" >&2 exit 1 fi done if ! grep -qxF -- "mutation_gate: could not write to GITHUB_STEP_SUMMARY" "$work/err.log"; then - echo "gate_unwritable_summary_test: FAIL: stderr lacks the could-not-write note" >&2 + echo "gate_summary_error_test: FAIL: stderr lacks the could-not-write note" >&2 cat "$work/err.log" >&2 exit 1 fi """ -def _unwritable_impl(ctx): +def _summary_error_impl(ctx): gate = ctx.attr.gate[DefaultInfo].files_to_run.executable script = ctx.actions.declare_file(ctx.label.name + ".sh") ctx.actions.write( output = script, - content = _UNWRITABLE_SCRIPT.format( + content = _SUMMARY_ERROR_SCRIPT.format( gate = gate.short_path, status = ctx.attr.status, lines = " ".join([shell.quote(line) for line in ctx.attr.lines]), @@ -116,8 +116,8 @@ def _unwritable_impl(ctx): executable, runfiles = shell_executable(ctx, script, runfiles) return [DefaultInfo(executable = executable, runfiles = runfiles)] -gate_unwritable_summary_test = rule( - implementation = _unwritable_impl, +gate_summary_error_test = rule( + implementation = _summary_error_impl, test = True, toolchains = SHELL_TOOLCHAINS, doc = """Passes when `gate`, run with `GITHUB_STEP_SUMMARY` in a missing directory, From 74259beb3cd578a981c00916a10f477748e5094d Mon Sep 17 00:00:00 2001 From: "tbhb.dev Agent" Date: Sat, 14 Nov 2026 09:58:42 -0500 Subject: [PATCH 3/3] fix(quality): tolerate closed stderr and tighten gate wording The summary warning's own write to stderr could end the gate before its verdict when stderr was closed. Ignore that failure, require the FAIL line in the failing summary test, say no mutant failed when none was checked, and reword the header. Generated-by: LLM effort=medium;harness=claude-code:2.1.318;model=anthropic:claude-sonnet-5-5;session=sha-256-64:06dc52093e5d8aba;surface=sdk-cli --- quality/private/mutation_gate.sh.tpl | 18 ++++++++++++----- quality/tests/BUILD.bazel | 30 ++++++++++++++++++++++++++++ quality/tests/gate_output_test.bzl | 27 +++++++++++++++++++------ 3 files changed, 64 insertions(+), 11 deletions(-) diff --git a/quality/private/mutation_gate.sh.tpl b/quality/private/mutation_gate.sh.tpl index 9da9345..eaa0f85 100644 --- a/quality/private/mutation_gate.sh.tpl +++ b/quality/private/mutation_gate.sh.tpl @@ -24,8 +24,9 @@ # # A timed-out mutant counts as killed and stays listed (its mutant.timeout or # mutant.timed_out lines print). The count line goes to stdout, and to -# $GITHUB_STEP_SUMMARY when that is set; a failed write there only warns on -# stderr. +# $GITHUB_STEP_SUMMARY when that variable is set and writable; a failed write +# there only warns on stderr, and delivery isn't guaranteed under a sandbox or +# with a cached test result. # # Fails on: any lived, not_covered or errored mutant; timeouts above # MAX_TIMEOUT_PERCENT of total, in the sum and in any one report; an @@ -200,9 +201,10 @@ if ((timeout > 0 && total > 0)); then # mutmut says timeout, gomutants timed_out; the lines print as reported. printf '%s\n' "$mutants" | grep -E '^mutant\.(timeout|timed_out)=' || true if [[ -n ${GITHUB_STEP_SUMMARY:-} ]]; then - # A summary that cannot be written must not end the run before the verdict. + # A summary that cannot be written must not end the run before the verdict, + # even with stderr closed, so the warning's own failure is ignored too. echo "$timeout_line" >>"$GITHUB_STEP_SUMMARY" || - echo "mutation_gate: could not write to GITHUB_STEP_SUMMARY" >&2 + echo "mutation_gate: could not write to GITHUB_STEP_SUMMARY" >&2 || true fi fi @@ -240,4 +242,10 @@ if ((${#problems[@]} > 0)); then fi exit 1 fi -echo "mutation_gate: ok: every mutant was killed" +# Only claim kills when some mutant was killed or timed out; an empty or +# all-skipped report (a diff that selected none) checked nothing. +if ((killed + timeout > 0)); then + echo "mutation_gate: ok: every mutant was killed" +else + echo "mutation_gate: ok: no mutant failed (none were checked)" +fi diff --git a/quality/tests/BUILD.bazel b/quality/tests/BUILD.bazel index a2e43cd..ee46509 100644 --- a/quality/tests/BUILD.bazel +++ b/quality/tests/BUILD.bazel @@ -89,6 +89,22 @@ PROOF_REASONS = { for proof, pattern in PROOF_REASONS.items() ] +# A diff-scoped report with no selected mutants checked nothing, so the gate +# passes without saying every mutant was killed. +mutation_gate_test( + name = "diff_empty_gate", + reports = ["testdata/diff_empty.txt"], + tags = PROOF_TAGS, +) + +gate_output_test( + name = "diff_empty_output_test", + gate = ":diff_empty_gate", + lines = [ + "mutation_gate: ok: no mutant failed (none were checked)", + ], +) + # Timeouts count as killed up to 10% of the mutants (10 of 100 passes, 5 # passes, 11 fails), stay listed, and are counted on stdout and in the job # summary. Gate failures list them too. @@ -139,6 +155,7 @@ gate_summary_error_test( gate_summary_error_test( name = "summary_error_fails_test", + err_lines = ["mutation_gate: FAIL: 1 lived"], gate = ":proof_timeout_with_survivor", lines = [ "mutation_gate: 5 timed out of 100 (5.0%), counted as killed", @@ -147,6 +164,19 @@ gate_summary_error_test( status = 1, ) +# With stderr closed too, the warning's own write fails and still must not end +# the run before the verdict. +gate_summary_error_test( + name = "summary_error_closed_stderr_test", + close_stderr = True, + gate = ":timeout_below_cap_gate", + lines = [ + "mutation_gate: 5 timed out of 100 (5.0%), counted as killed", + "mutation_gate: ok: every mutant was killed", + ], + status = 0, +) + # Timed-out mutants are listed in the gate's output, passing or failing. expect_failure_test( name = "proof_timeout_with_survivor_lists_timeouts_fails", diff --git a/quality/tests/gate_output_test.bzl b/quality/tests/gate_output_test.bzl index f0d4028..03be0db 100644 --- a/quality/tests/gate_output_test.bzl +++ b/quality/tests/gate_output_test.bzl @@ -80,7 +80,13 @@ gate="$PWD/{gate}" work="$TEST_TMPDIR/gate_summary_error" mkdir -p "$work" status=0 -GITHUB_STEP_SUMMARY="$work/missing/summary.md" "$gate" >"$work/out.log" 2>"$work/err.log" || status=$? +: >"$work/err.log" +if (({close_stderr})); then + # Nothing can be written to stderr, the warning included. + GITHUB_STEP_SUMMARY="$work/missing/summary.md" "$gate" >"$work/out.log" 2>&- || status=$? +else + GITHUB_STEP_SUMMARY="$work/missing/summary.md" "$gate" >"$work/out.log" 2>"$work/err.log" || status=$? +fi if ((status != {status})); then echo "gate_summary_error_test: FAIL: the gate exited $status, not {status}" >&2 cat "$work/out.log" "$work/err.log" >&2 @@ -93,10 +99,14 @@ for line in {lines}; do exit 1 fi done -if ! grep -qxF -- "mutation_gate: could not write to GITHUB_STEP_SUMMARY" "$work/err.log"; then - echo "gate_summary_error_test: FAIL: stderr lacks the could-not-write note" >&2 - cat "$work/err.log" >&2 - exit 1 +if ! (({close_stderr})); then + for line in "mutation_gate: could not write to GITHUB_STEP_SUMMARY" {err_lines}; do + if ! grep -qxF -- "$line" "$work/err.log"; then + echo "gate_summary_error_test: FAIL: stderr lacks '$line'" >&2 + cat "$work/err.log" >&2 + exit 1 + fi + done fi """ @@ -108,6 +118,8 @@ def _summary_error_impl(ctx): content = _SUMMARY_ERROR_SCRIPT.format( gate = gate.short_path, status = ctx.attr.status, + close_stderr = 1 if ctx.attr.close_stderr else 0, + err_lines = " ".join([shell.quote(line) for line in ctx.attr.err_lines]), lines = " ".join([shell.quote(line) for line in ctx.attr.lines]), ), is_executable = True, @@ -122,8 +134,11 @@ gate_summary_error_test = rule( toolchains = SHELL_TOOLCHAINS, doc = """Passes when `gate`, run with `GITHUB_STEP_SUMMARY` in a missing directory, still exits with `status`, prints each of `lines` to stdout and notes the -failed write on stderr.""", +failed write on stderr, with each of `err_lines`. With `close_stderr` the gate +runs with stderr closed and nothing is read from it.""", attrs = SHELL_ATTRS | { + "close_stderr": attr.bool(default = False, doc = "Run the gate with stderr closed."), + "err_lines": attr.string_list(doc = "More whole lines the gate's stderr must hold."), "gate": attr.label(mandatory = True, executable = True, cfg = "target"), "lines": attr.string_list(mandatory = True, doc = "Whole lines the gate's stdout must hold."), "status": attr.int(mandatory = True, values = [0, 1], doc = "The exit status the gate must keep."),