Skip to content

fix(action): enforce truthful quality-gate outcomes #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
roli-lpci merged 1 commit into from
Aug 17, 2026
Merged

fix(action): enforce truthful quality-gate outcomes #20

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 45 additions & 42 deletions .github/workflows/action-smoke.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,57 +6,43 @@ on:
- main
- "codex/pyg-mp-*"
workflow_dispatch:
inputs:
scenario:
description: Fixture outcome to exercise
required: true
default: pass
type: choice
options:
- pass
- fail

permissions:
contents: read

jobs:
clean-consumer:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- case: pass-default
fixture: pass
observation: false
expected-outcome: success
expected-status: pass
- case: fail-default
fixture: fail
observation: false
expected-outcome: failure
expected-status: fail
- case: fail-observation
fixture: fail
observation: true
expected-outcome: success
expected-status: fail
steps:
- name: Check out the action source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
path: action-source
persist-credentials: false

- name: Select fixture scenario
id: fixture
shell: bash
env:
EVENT_NAME: ${{ github.event_name }}
GIT_REF: ${{ github.ref }}
REQUESTED_SCENARIO: ${{ inputs.scenario }}
run: |
# The action currently owns a fixed artifact name, so each workflow
# run exercises exactly one scenario without cross-job collisions.
if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then
scenario="$REQUESTED_SCENARIO"
elif [[ "$GIT_REF" == "refs/heads/main" ]]; then
scenario="fail"
else
scenario="pass"
fi

if [[ "$scenario" != "pass" && "$scenario" != "fail" ]]; then
echo "Unsupported smoke scenario: $scenario" >&2
exit 1
fi
echo "scenario=$scenario" >> "$GITHUB_OUTPUT"

- name: Build a clean consumer workspace
shell: bash
env:
SCENARIO: ${{ steps.fixture.outputs.scenario }}
SCENARIO: ${{ matrix.fixture }}
run: |
action_source="$RUNNER_TEMP/pygate-action-source-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
fixture_staging="$(mktemp -d "$RUNNER_TEMP/pygate-action-fixture.XXXXXX")"
Expand Down Expand Up @@ -94,32 +80,49 @@ jobs:
test -f "$action_path/action.yml"
test -z "$(git status --porcelain)"

- name: Run the checked-out root action
id: gate
- name: Run the checked-out root action with defaults
id: gate-default
if: ${{ !matrix.observation }}
continue-on-error: true
uses: ./action-source
with:
mode: canary
repair: "false"
python-version: "3.12"
post-comment: "false"

- name: Run the checked-out root action in observation mode
id: gate-observation
if: ${{ matrix.observation }}
continue-on-error: true
uses: ./action-source
with:
mode: canary
repair: "false"
python-version: "3.12"
post-comment: "false"
fail-on-error: "false"

- name: Record and assert the declared status
- name: Record and assert the action outcome and declared status
if: always()
shell: bash
env:
ACTUAL_STATUS: ${{ steps.gate.outputs.status }}
EXPECTED_STATUS: ${{ steps.fixture.outputs.scenario }}
ACTUAL_OUTCOME: ${{ matrix.observation && steps.gate-observation.outcome || steps.gate-default.outcome }}
ACTUAL_STATUS: ${{ matrix.observation && steps.gate-observation.outputs.status || steps.gate-default.outputs.status }}
EXPECTED_OUTCOME: ${{ matrix.expected-outcome }}
EXPECTED_STATUS: ${{ matrix.expected-status }}
run: |
printf 'Expected PyGate status: %s\nDeclared PyGate status: %s\n' \
"$EXPECTED_STATUS" "$ACTUAL_STATUS" | tee -a "$GITHUB_STEP_SUMMARY"
printf 'Expected action outcome: %s\nActual action outcome: %s\nExpected PyGate status: %s\nDeclared PyGate status: %s\n' \
"$EXPECTED_OUTCOME" "$ACTUAL_OUTCOME" "$EXPECTED_STATUS" "$ACTUAL_STATUS" | tee -a "$GITHUB_STEP_SUMMARY"
test "$ACTUAL_OUTCOME" = "$EXPECTED_OUTCOME"
test "$ACTUAL_STATUS" = "$EXPECTED_STATUS"
test -f .pygate/gate-result.json

- name: Upload scenario evidence
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: pygate-smoke-${{ steps.fixture.outputs.scenario }}-${{ github.run_attempt }}
name: pygate-smoke-${{ matrix.case }}-${{ github.run_attempt }}
path: .pygate/
if-no-files-found: error
include-hidden-files: true
Expand Down
93 changes: 78 additions & 15 deletions action.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ inputs:
post-comment:
description: "Post findings as PR comment"
default: "true"
fail-on-error:
description: "Fail the action when quality gates fail or repair does not pass; set to false for observation-only reporting"
default: "true"

outputs:
status:
Expand Down Expand Up @@ -70,18 +73,29 @@ runs:
echo "::error::Invalid mode: $PYGATE_MODE (must be canary or full)"
exit 1
fi
rm -f .pygate/gate-result.json .pygate/failures.json
set +e
pygate run --mode "$PYGATE_MODE" --changed-files .pygate-changed-files.txt --output-dir .pygate
EXIT_CODE=$?
set -e
if [ $EXIT_CODE -eq 0 ]; then
echo "status=pass" >> "$GITHUB_OUTPUT"
else
echo "status=fail" >> "$GITHUB_OUTPUT"
fi
if [ -f .pygate/failures.json ]; then
echo "failures_json=.pygate/failures.json" >> "$GITHUB_OUTPUT"
fi
case "$EXIT_CODE" in
0|1)
if [ ! -f .pygate/gate-result.json ] || [ ! -f .pygate/failures.json ]; then
echo "::error::PyGate exited $EXIT_CODE without writing its required result artifacts"
exit 2
fi
if [ "$EXIT_CODE" -eq 0 ]; then
echo "status=pass" >> "$GITHUB_OUTPUT"
else
echo "status=fail" >> "$GITHUB_OUTPUT"
fi
;;
*)
echo "::error::PyGate command failed with infrastructure exit code $EXIT_CODE"
exit "$EXIT_CODE"
;;
esac
echo "failures_json=.pygate/failures.json" >> "$GITHUB_OUTPUT"

- name: Run bounded repair
id: repair
Expand All @@ -94,13 +108,18 @@ runs:
pygate repair --input .pygate/failures.json --max-attempts "$PYGATE_MAX_ATTEMPTS"
EXIT_CODE=$?
set -e
if [ $EXIT_CODE -eq 0 ]; then
echo "status=pass" >> "$GITHUB_OUTPUT"
elif [ $EXIT_CODE -eq 2 ]; then
echo "status=escalated" >> "$GITHUB_OUTPUT"
else
echo "status=fail" >> "$GITHUB_OUTPUT"
fi
case "$EXIT_CODE" in
0)
echo "status=pass" >> "$GITHUB_OUTPUT"
;;
2)
echo "status=escalated" >> "$GITHUB_OUTPUT"
;;
*)
echo "::error::PyGate repair command failed with infrastructure exit code $EXIT_CODE"
exit "$EXIT_CODE"
;;
esac

- name: Set repair status to skipped
id: repair-skip
Expand Down Expand Up @@ -139,3 +158,47 @@ runs:
name: pygate-artifacts
path: .pygate/
if-no-files-found: ignore

- name: Enforce action outcome
if: always()
shell: bash
env:
FAIL_ON_ERROR: ${{ inputs.fail-on-error }}
GATE_STATUS: ${{ steps.gate.outputs.status }}
REPAIR_STATUS: ${{ steps.repair.outputs.status || steps.repair-skip.outputs.status }}
run: |
case "$FAIL_ON_ERROR" in
true|false)
;;
*)
echo "::error::Invalid fail-on-error value: $FAIL_ON_ERROR (must be true or false)"
exit 2
;;
esac

case "$GATE_STATUS" in
pass|fail)
;;
*)
echo "::error::Quality gate did not produce a valid status"
exit 2
;;
esac

case "$REPAIR_STATUS" in
pass|fail|escalated|skipped)
;;
*)
echo "::error::Repair did not produce a valid status"
exit 2
;;
esac

if [[ "$FAIL_ON_ERROR" == "false" ]]; then
exit 0
fi
if [[ "$GATE_STATUS" == "fail" || "$REPAIR_STATUS" == "fail" || "$REPAIR_STATUS" == "escalated" ]]; then
echo "::error::PyGate quality gates did not pass"
exit 1
fi
exit 0
Loading