diff --git a/.github/workflows/action-smoke.yml b/.github/workflows/action-smoke.yml index 8d933be..4119ec4 100644 --- a/.github/workflows/action-smoke.yml +++ b/.github/workflows/action-smoke.yml @@ -5,6 +5,7 @@ on: branches: - main - "codex/pyg-mp-*" + - "agent/pygate-*" workflow_dispatch: permissions: @@ -19,19 +20,44 @@ jobs: include: - case: pass-default fixture: pass - observation: false + repair: false + fail-on-error: true expected-outcome: success + expected-gate-status: pass expected-status: pass + expected-repair-status: skipped - case: fail-default fixture: fail - observation: false + repair: false + fail-on-error: true expected-outcome: failure + expected-gate-status: fail expected-status: fail + expected-repair-status: skipped - case: fail-observation fixture: fail - observation: true + repair: false + fail-on-error: false expected-outcome: success + expected-gate-status: fail expected-status: fail + expected-repair-status: skipped + - case: repair-success + fixture: repair + repair: true + fail-on-error: true + expected-outcome: success + expected-gate-status: fail + expected-status: pass + expected-repair-status: pass + - case: repair-escalation + fixture: fail + repair: true + fail-on-error: true + expected-outcome: failure + expected-gate-status: fail + expected-status: escalated + expected-repair-status: escalated steps: - name: Check out the action source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -47,7 +73,11 @@ jobs: action_source="$RUNNER_TEMP/pygate-action-source-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" fixture_staging="$(mktemp -d "$RUNNER_TEMP/pygate-action-fixture.XXXXXX")" - cp "$GITHUB_WORKSPACE/action-source/tests/action-fixture/pyproject.toml" "$fixture_staging/" + if [[ "$SCENARIO" == "repair" ]]; then + cp "$GITHUB_WORKSPACE/action-source/tests/action-fixture/repair-pyproject.toml" "$fixture_staging/pyproject.toml" + else + cp "$GITHUB_WORKSPACE/action-source/tests/action-fixture/pyproject.toml" "$fixture_staging/" + fi cp "$GITHUB_WORKSPACE/action-source/tests/action-fixture/$SCENARIO.py" "$fixture_staging/check.py" mv "$GITHUB_WORKSPACE/action-source" "$action_source" @@ -80,50 +110,64 @@ jobs: test -f "$action_path/action.yml" test -z "$(git status --porcelain)" - - name: Run the checked-out root action with defaults - id: gate-default - if: ${{ !matrix.observation }} + - name: Run the checked-out root action + id: gate continue-on-error: true uses: ./action-source with: mode: canary - repair: "false" + repair: ${{ matrix.repair }} python-version: "3.12" post-comment: "false" - - - name: Run the checked-out root action in observation mode - id: gate-observation - if: ${{ matrix.observation }} - continue-on-error: true - uses: ./action-source - with: - mode: canary - repair: "false" - python-version: "3.12" - post-comment: "false" - fail-on-error: "false" + fail-on-error: ${{ matrix.fail-on-error }} + artifact-name: pygate-action-${{ matrix.case }}-${{ github.run_attempt }} - name: Record and assert the action outcome and declared status if: always() shell: bash env: - ACTUAL_OUTCOME: ${{ matrix.observation && steps.gate-observation.outcome || steps.gate-default.outcome }} - ACTUAL_STATUS: ${{ matrix.observation && steps.gate-observation.outputs.status || steps.gate-default.outputs.status }} + ACTUAL_OUTCOME: ${{ steps.gate.outcome }} + ACTUAL_GATE_STATUS: ${{ steps.gate.outputs.gate-status }} + ACTUAL_STATUS: ${{ steps.gate.outputs.status }} + ACTUAL_REPAIR_STATUS: ${{ steps.gate.outputs.repair-status }} EXPECTED_OUTCOME: ${{ matrix.expected-outcome }} + EXPECTED_GATE_STATUS: ${{ matrix.expected-gate-status }} EXPECTED_STATUS: ${{ matrix.expected-status }} + EXPECTED_REPAIR_STATUS: ${{ matrix.expected-repair-status }} run: | - printf 'Expected action outcome: %s\nActual action outcome: %s\nExpected PyGate status: %s\nDeclared PyGate status: %s\n' \ - "$EXPECTED_OUTCOME" "$ACTUAL_OUTCOME" "$EXPECTED_STATUS" "$ACTUAL_STATUS" | tee -a "$GITHUB_STEP_SUMMARY" + printf 'Expected action outcome: %s\nActual action outcome: %s\nExpected gate status: %s\nDeclared gate status: %s\nExpected final status: %s\nDeclared final status: %s\nExpected repair status: %s\nDeclared repair status: %s\n' \ + "$EXPECTED_OUTCOME" "$ACTUAL_OUTCOME" "$EXPECTED_GATE_STATUS" "$ACTUAL_GATE_STATUS" "$EXPECTED_STATUS" "$ACTUAL_STATUS" "$EXPECTED_REPAIR_STATUS" "$ACTUAL_REPAIR_STATUS" | tee -a "$GITHUB_STEP_SUMMARY" test "$ACTUAL_OUTCOME" = "$EXPECTED_OUTCOME" + test "$ACTUAL_GATE_STATUS" = "$EXPECTED_GATE_STATUS" test "$ACTUAL_STATUS" = "$EXPECTED_STATUS" + test "$ACTUAL_REPAIR_STATUS" = "$EXPECTED_REPAIR_STATUS" test -f .pygate/gate-result.json + test -f .pygate/failures.json + jq -e '.schema == "gate-result/v1" and (.status | IN("pass", "fail", "timeout", "error"))' .pygate/gate-result.json >/dev/null - - name: Upload scenario evidence - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + verify-action-artifacts: + if: always() + needs: clean-consumer + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + case: + - pass-default + - fail-default + - fail-observation + - repair-success + - repair-escalation + steps: + - name: Download action-owned artifact + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: pygate-smoke-${{ matrix.case }}-${{ github.run_attempt }} - path: .pygate/ - if-no-files-found: error - include-hidden-files: true - retention-days: 7 + name: pygate-action-${{ matrix.case }}-${{ github.run_attempt }} + path: artifact + + - name: Assert hidden action evidence is present + shell: bash + run: | + test -f artifact/gate-result.json + test -f artifact/failures.json + jq -e '.schema == "gate-result/v1"' artifact/gate-result.json >/dev/null diff --git a/action.yml b/action.yml index f46c90c..f4f5eab 100644 --- a/action.yml +++ b/action.yml @@ -22,13 +22,19 @@ inputs: post-comment: description: "Post findings as PR comment" default: "true" + artifact-name: + description: "Name for the uploaded PyGate artifact bundle" + default: "pygate-artifacts" fail-on-error: description: "Fail the action when quality gates fail or repair does not pass; set to false for observation-only reporting" default: "true" outputs: status: - description: "Overall gate result: pass, fail, or escalated" + description: "Final result after the quality gate and optional repair: pass, fail, or escalated" + value: ${{ steps.final.outputs.status }} + gate-status: + description: "Initial quality-gate result before optional repair: pass or fail" value: ${{ steps.gate.outputs.status }} failures-json: description: "Path to .pygate/failures.json" @@ -116,6 +122,7 @@ runs: echo "status=escalated" >> "$GITHUB_OUTPUT" ;; *) + echo "status=fail" >> "$GITHUB_OUTPUT" echo "::error::PyGate repair command failed with infrastructure exit code $EXIT_CODE" exit "$EXIT_CODE" ;; @@ -152,52 +159,109 @@ runs: }); - name: Upload artifacts + id: artifacts if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: pygate-artifacts + name: ${{ inputs.artifact-name }} path: .pygate/ + include-hidden-files: true if-no-files-found: ignore - - name: Enforce action outcome + - name: Set final action status + id: final if: always() shell: bash env: - FAIL_ON_ERROR: ${{ inputs.fail-on-error }} GATE_STATUS: ${{ steps.gate.outputs.status }} + GATE_OUTCOME: ${{ steps.gate.outcome }} REPAIR_STATUS: ${{ steps.repair.outputs.status || steps.repair-skip.outputs.status }} + REPAIR_OUTCOME: ${{ steps.repair.outcome }} + ARTIFACT_OUTCOME: ${{ steps.artifacts.outcome }} run: | - case "$FAIL_ON_ERROR" in - true|false) + infrastructure_failure=false + final_status=fail + + case "$GATE_STATUS" in + pass|fail) ;; *) - echo "::error::Invalid fail-on-error value: $FAIL_ON_ERROR (must be true or false)" - exit 2 + infrastructure_failure=true ;; esac - case "$GATE_STATUS" in - pass|fail) + if [[ "$GATE_OUTCOME" != "success" ]]; then + infrastructure_failure=true + fi + if [[ "$REPAIR_OUTCOME" == "failure" ]]; then + infrastructure_failure=true + fi + if [[ "$ARTIFACT_OUTCOME" != "success" ]]; then + infrastructure_failure=true + fi + + if [[ "$infrastructure_failure" == "false" ]]; then + case "$GATE_STATUS:$REPAIR_STATUS" in + pass:skipped|pass:pass) + final_status=pass + ;; + fail:pass) + final_status=pass + ;; + fail:escalated) + final_status=escalated + ;; + fail:fail|fail:skipped) + final_status=fail + ;; + *) + infrastructure_failure=true + ;; + esac + fi + + echo "status=$final_status" >> "$GITHUB_OUTPUT" + echo "infrastructure_failure=$infrastructure_failure" >> "$GITHUB_OUTPUT" + echo "Final action status: $final_status" + echo "Initial gate status: ${GATE_STATUS:-unavailable}" + echo "Repair status: ${REPAIR_STATUS:-unavailable}" + echo "Infrastructure failure: $infrastructure_failure" + + - name: Enforce action outcome + if: always() + shell: bash + env: + FAIL_ON_ERROR: ${{ inputs.fail-on-error }} + FINAL_STATUS: ${{ steps.final.outputs.status }} + INFRASTRUCTURE_FAILURE: ${{ steps.final.outputs.infrastructure_failure }} + run: | + case "$FAIL_ON_ERROR" in + true|false) ;; *) - echo "::error::Quality gate did not produce a valid status" + echo "::error::Invalid fail-on-error value: $FAIL_ON_ERROR (must be true or false)" exit 2 ;; esac - case "$REPAIR_STATUS" in - pass|fail|escalated|skipped) + case "$FINAL_STATUS" in + pass|fail|escalated) ;; *) - echo "::error::Repair did not produce a valid status" + echo "::error::Final action status was not produced" exit 2 ;; esac + if [[ "$INFRASTRUCTURE_FAILURE" == "true" ]]; then + echo "::error::PyGate action failed before producing a trustworthy quality result" + exit 1 + fi + if [[ "$FAIL_ON_ERROR" == "false" ]]; then exit 0 fi - if [[ "$GATE_STATUS" == "fail" || "$REPAIR_STATUS" == "fail" || "$REPAIR_STATUS" == "escalated" ]]; then + if [[ "$FINAL_STATUS" == "fail" || "$FINAL_STATUS" == "escalated" ]]; then echo "::error::PyGate quality gates did not pass" exit 1 fi diff --git a/tests/action-fixture/pyproject.toml b/tests/action-fixture/pyproject.toml index a5b2e1f..59908bc 100644 --- a/tests/action-fixture/pyproject.toml +++ b/tests/action-fixture/pyproject.toml @@ -2,6 +2,9 @@ target-version = "py312" exclude = [".pygate", "action-source"] +[tool.ruff.lint] +ignore = ["RUF022"] + [tool.pyright] pythonVersion = "3.12" typeCheckingMode = "basic" diff --git a/tests/action-fixture/repair-pyproject.toml b/tests/action-fixture/repair-pyproject.toml new file mode 100644 index 0000000..2982445 --- /dev/null +++ b/tests/action-fixture/repair-pyproject.toml @@ -0,0 +1,12 @@ +[tool.ruff] +target-version = "py312" +exclude = [".pygate", "action-source"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"] + +[tool.pyright] +pythonVersion = "3.12" +typeCheckingMode = "basic" +include = ["check.py"] +exclude = [".pygate", "action-source"] diff --git a/tests/action-fixture/repair.py b/tests/action-fixture/repair.py new file mode 100644 index 0000000..99b7082 --- /dev/null +++ b/tests/action-fixture/repair.py @@ -0,0 +1,9 @@ +__all__ = ["other", "answer"] + + +def answer() -> int: + return 42 + + +def other() -> int: + return 7