diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acb8b4e09..9d79d3d18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ name: CI +run-name: ${{ github.event_name == 'pull_request' && format('CI / {0}', github.event.pull_request.number) || '' }} on: push: @@ -8,6 +9,12 @@ on: types: [opened, synchronize, reopened, edited] workflow_dispatch: +# PR metadata edits can retrigger full CI for the same head. Keep only the +# newest run for a pull request; push and manual runs use a unique run ID. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read actions: read @@ -295,6 +302,11 @@ jobs: e2e-dry-run: needs: [unit-test, lint, script-test, deterministic-gate] runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + mode: ${{ steps.e2e_domains.outputs.mode }} + reason: ${{ steps.e2e_domains.outputs.reason }} + live_packages: ${{ steps.e2e_domains.outputs.live_packages }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 with: @@ -308,6 +320,23 @@ jobs: - name: Resolve CLI E2E domains id: e2e_domains run: node scripts/e2e_domains.js + - name: Validate CLI E2E domain outputs + env: + E2E_MODE: ${{ steps.e2e_domains.outputs.mode }} + E2E_LIVE_PACKAGES: ${{ steps.e2e_domains.outputs.live_packages }} + run: | + case "$E2E_MODE" in + skip) + [ -z "$E2E_LIVE_PACKAGES" ] || { echo "::error::Skip mode must not resolve live packages"; exit 1; } + ;; + full|subset) + [ -n "$E2E_LIVE_PACKAGES" ] || { echo "::error::No live packages resolved for mode $E2E_MODE"; exit 1; } + ;; + *) + echo "::error::Invalid CLI E2E mode: $E2E_MODE" + exit 1 + ;; + esac - name: Build lark-cli if: ${{ steps.e2e_domains.outputs.mode != 'skip' }} run: make build @@ -341,10 +370,17 @@ jobs: fi e2e-live: - needs: [unit-test, lint, script-test, deterministic-gate] - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork }} + needs: [unit-test, lint, script-test, deterministic-gate, e2e-dry-run] + if: ${{ always() && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) && needs.unit-test.result == 'success' && needs.lint.result == 'success' && needs.script-test.result == 'success' && needs.deterministic-gate.result == 'success' && needs.e2e-dry-run.result == 'success' && (needs.e2e-dry-run.outputs.mode == 'full' || needs.e2e-dry-run.outputs.mode == 'subset') && needs.e2e-dry-run.outputs.live_packages != '' }} runs-on: ubuntu-latest + timeout-minutes: 30 + # Live E2E uses one repository-wide execution slot. + concurrency: + group: lark-cli-e2e-live + cancel-in-progress: false + queue: max permissions: + actions: read contents: read checks: write env: @@ -360,31 +396,46 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: '3.x' - - name: Resolve CLI E2E domains - id: e2e_domains - run: node scripts/e2e_domains.js - name: Build lark-cli - if: ${{ steps.e2e_domains.outputs.mode != 'skip' }} + id: build_cli run: make build - name: Prepare shared live E2E tenant token id: live_e2e_tat - if: ${{ steps.e2e_domains.outputs.mode != 'skip' }} env: LARKSUITE_CLI_APP_ID: ${{ secrets.TEST_BOT1_APP_ID }} TEST_BOT1_APP_SECRET: ${{ secrets.TEST_BOT1_APP_SECRET }} run: node scripts/fetch_e2e_tat.js - name: Run CLI E2E tests + # Keep an active Go test alive so t.Cleanup can finish. A queued stale + # run is rejected below before it can start live E2E. + if: ${{ always() && steps.build_cli.outcome == 'success' && steps.live_e2e_tat.outcome == 'success' }} + shell: bash env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + EVENT_NAME: ${{ github.event_name }} + RUN_ID: ${{ github.run_id }} + RUN_NUMBER: ${{ github.run_number }} + RUN_GENERATION: ${{ github.event_name == 'pull_request' && format('CI / {0}', github.event.pull_request.number) || '' }} LARK_CLI_BIN: ${{ github.workspace }}/lark-cli - E2E_MODE: ${{ steps.e2e_domains.outputs.mode }} - E2E_REASON: ${{ steps.e2e_domains.outputs.reason }} - E2E_LIVE_PACKAGES: ${{ steps.e2e_domains.outputs.live_packages }} + E2E_MODE: ${{ needs.e2e-dry-run.outputs.mode }} + E2E_REASON: ${{ needs.e2e-dry-run.outputs.reason }} + E2E_LIVE_PACKAGES: ${{ needs.e2e-dry-run.outputs.live_packages }} E2E_TENANT_AUTH_FILE: ${{ steps.live_e2e_tat.outputs.path }} TEST_USER_ACCESS_TOKEN: ${{ secrets.TEST_USER_ACCESS_TOKEN }} run: | - if [ "$E2E_MODE" = "skip" ]; then - echo "No live CLI E2E needed: $E2E_REASON" - exit 0 + if [ "$EVENT_NAME" = "pull_request" ]; then + workflow_id="$(gh api "repos/$REPOSITORY/actions/runs/$RUN_ID" --jq '.workflow_id')" + newer_runs="$( + gh api --paginate -X GET "repos/$REPOSITORY/actions/workflows/$workflow_id/runs" \ + -f event=pull_request -f branch="$GITHUB_HEAD_REF" -f per_page=100 | + jq -r --arg repository "$REPOSITORY" --arg generation "$RUN_GENERATION" --argjson run_number "$RUN_NUMBER" \ + '.workflow_runs[] | select(.head_repository.full_name == $repository and .display_title == $generation and .run_number > $run_number) | .id' + )" + if [ -n "$newer_runs" ]; then + echo "::error::Superseded before live E2E started by newer workflow run(s): $newer_runs" + exit 1 + fi fi if [ -z "${E2E_TENANT_AUTH_FILE:-}" ] || [ ! -f "$E2E_TENANT_AUTH_FILE" ]; then echo "::error::Missing shared live E2E tenant token file" @@ -416,7 +467,7 @@ jobs: echo "Live CLI E2E packages: $packages" go run gotest.tools/gotestsum@v1.12.3 --rerun-fails=2 --rerun-fails-max-failures=20 --packages="$packages" --format testname --junitfile cli-e2e-report.xml -- -count=1 -v - name: Publish CLI E2E test report - if: ${{ !cancelled() && steps.e2e_domains.outputs.mode != 'skip' }} + if: ${{ !cancelled() }} uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0 with: name: CLI E2E Tests @@ -493,8 +544,8 @@ jobs: echo "| L4 | sidecar-integration (observe-only) | ${{ needs.sidecar-integration.result }} |" >> $GITHUB_STEP_SUMMARY # Any failure or cancellation in any job blocks the merge. - # Legitimately skipped jobs (deadcode on push, e2e-live on fork, - # license-header on push) are OK. + # Legitimately skipped jobs (deadcode on push, e2e-live when not + # needed or on a fork, license-header on push) are OK. # # plugin-integration and sidecar-integration are intentionally NOT # in this loop yet: they run on every PR and their status is shown diff --git a/scripts/ci-workflow.test.sh b/scripts/ci-workflow.test.sh index f5cc7e5a5..6acdb0adc 100644 --- a/scripts/ci-workflow.test.sh +++ b/scripts/ci-workflow.test.sh @@ -18,6 +18,11 @@ workflow_permissions="$(awk ' in_permissions && /^[^[:space:]]/ { exit } in_permissions { print } ' "$workflow")" +workflow_concurrency="$(awk ' + /^concurrency:/ { in_concurrency = 1; print; next } + in_concurrency && /^[^[:space:]]/ { exit } + in_concurrency { print } +' "$workflow")" fast_gate_section="$(job_section fast-gate)" unit_test_section="$(job_section unit-test)" lint_section="$(awk ' @@ -46,6 +51,27 @@ results_section="$(awk ' in_job { print } ' "$workflow")" fork_safe_guard="github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork" +live_job_condition="always() && ($fork_safe_guard) && needs.unit-test.result == 'success' && needs.lint.result == 'success' && needs.script-test.result == 'success' && needs.deterministic-gate.result == 'success' && needs.e2e-dry-run.result == 'success' && (needs.e2e-dry-run.outputs.mode == 'full' || needs.e2e-dry-run.outputs.mode == 'subset') && needs.e2e-dry-run.outputs.live_packages != ''" + +if ! grep -Fq "run-name: \${{ github.event_name == 'pull_request' && format('CI / {0}', github.event.pull_request.number) || '' }}" "$workflow"; then + echo "CI should expose a stable PR generation while preserving default push and manual run titles" >&2 + exit 1 +fi + +if ! grep -Fq "RUN_GENERATION: \${{ github.event_name == 'pull_request' && format('CI / {0}', github.event.pull_request.number) || '' }}" <<<"$section"; then + echo "the supersession generation should match the PR-only run name" >&2 + exit 1 +fi + +if ! grep -Fq 'group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}' <<<"$workflow_concurrency"; then + echo "CI should deduplicate runs for the same pull request without grouping push or manual runs" >&2 + exit 1 +fi + +if ! grep -Fq "cancel-in-progress: \${{ github.event_name == 'pull_request' }}" <<<"$workflow_concurrency"; then + echo "CI should cancel superseded pull request runs but preserve push and manual runs" >&2 + exit 1 +fi for denied_permission in "checks: write" "pull-requests: write" "issues: write"; do if grep -Eq "^[[:space:]]*${denied_permission}$" <<<"$workflow_permissions"; then @@ -210,8 +236,84 @@ if ! grep -Fq "deterministic-gate" <<<"$results_section"; then exit 1 fi -if ! grep -Fq "if: \${{ $fork_safe_guard }}" <<<"$section"; then - echo "e2e-live should run on push and same-repository pull_request, but skip fork pull_request" +if ! grep -Fq "if: \${{ $live_job_condition }}" <<<"$section"; then + echo "e2e-live should preserve active cleanup while requiring a successful non-skip dry run and excluding fork pull requests" + exit 1 +fi + +if ! grep -Fq "needs: [unit-test, lint, script-test, deterministic-gate, e2e-dry-run]" <<<"$section"; then + echo "e2e-live should wait outside the exclusive queue until e2e-dry-run finishes" + exit 1 +fi + +if ! grep -Fq "timeout-minutes: 20" <<<"$dry_run_section"; then + echo "e2e-dry-run should bound the planning gate before live E2E" >&2 + exit 1 +fi + +if ! grep -Fq "timeout-minutes: 30" <<<"$section"; then + echo "e2e-live should release the repository-wide slot after 30 minutes" >&2 + exit 1 +fi + +if ! grep -Fq "group: lark-cli-e2e-live" <<<"$section"; then + echo "e2e-live should use one repository-wide execution slot" >&2 + exit 1 +fi + +if ! grep -Fq "cancel-in-progress: false" <<<"$section"; then + echo "e2e-live should queue waiting runs instead of cancelling an active live test" >&2 + exit 1 +fi + +if ! grep -Fq "queue: max" <<<"$section"; then + echo "e2e-live should preserve queued runs instead of replacing an existing pending run" >&2 + exit 1 +fi + +if ! grep -Fq "actions: read" <<<"$section"; then + echo "e2e-live should use read-only Actions access for the supersession check" >&2 + exit 1 +fi + +live_test_step="$(awk ' + /^ - name: Run CLI E2E tests/ { in_step = 1 } + in_step { print } + in_step && /^ - name: Publish CLI E2E test report/ { exit } +' <<<"$section")" + +if ! grep -Fq "if: \${{ always() && steps.build_cli.outcome == 'success' && steps.live_e2e_tat.outcome == 'success' }}" <<<"$live_test_step"; then + echo "the active live test step should survive ordinary workflow supersession only after setup succeeds" >&2 + exit 1 +fi + +for required in \ + 'gh api "repos/$REPOSITORY/actions/runs/$RUN_ID"' \ + 'gh api --paginate -X GET "repos/$REPOSITORY/actions/workflows/$workflow_id/runs"' \ + '-f event=pull_request -f branch="$GITHUB_HEAD_REF" -f per_page=100' \ + '.head_repository.full_name == $repository and .display_title == $generation and .run_number > $run_number' \ + '::error::Superseded before live E2E started' \ + 'exit 1'; do + if ! grep -Fq -- "$required" <<<"$live_test_step"; then + echo "the live startup check should fail closed before a superseded run starts live E2E: missing $required" >&2 + exit 1 + fi +done + +if ! awk ' + /if \[ -n "\$newer_runs" \]; then/ { superseded_state = 1; next } + superseded_state == 1 && /::error::Superseded before live E2E started/ { superseded_state = 2; next } + superseded_state == 2 && /^[[:space:]]+exit 1[[:space:]]*$/ { superseded_state = 3; next } + superseded_state > 0 && /^[[:space:]]+fi[[:space:]]*$/ { + if (superseded_state != 3) exit 2 + superseded_closed = 1 + superseded_state = 0 + next + } + /go run gotest.tools\/gotestsum@/ { test_started = 1; if (!superseded_closed) exit 3 } + END { exit superseded_closed && test_started ? 0 : 1 } +' <<<"$live_test_step"; then + echo "a superseded live run must stop before gotestsum starts" >&2 exit 1 fi @@ -222,6 +324,39 @@ if ! grep -Fq "name: Resolve CLI E2E domains" <<<"$dry_run_section" || exit 1 fi +for output in \ + 'mode: ${{ steps.e2e_domains.outputs.mode }}' \ + 'reason: ${{ steps.e2e_domains.outputs.reason }}' \ + 'live_packages: ${{ steps.e2e_domains.outputs.live_packages }}'; do + if ! grep -Fq "$output" <<<"$dry_run_section"; then + echo "e2e-dry-run should publish $output for the live job" >&2 + exit 1 + fi +done + +for validation_contract in \ + 'case "$E2E_MODE" in' \ + 'skip)' \ + '[ -z "$E2E_LIVE_PACKAGES" ]' \ + 'full|subset)' \ + '[ -n "$E2E_LIVE_PACKAGES" ]' \ + 'Invalid CLI E2E mode' \ + 'exit 1'; do + if ! grep -Fq "$validation_contract" <<<"$dry_run_section"; then + echo "e2e-dry-run should fail invalid domain output before live can be skipped: missing $validation_contract" >&2 + exit 1 + fi +done + +if ! awk ' + /- name: Validate CLI E2E domain outputs/ { validated = 1 } + /- name: Build lark-cli/ { exit validated ? 0 : 1 } + END { if (!validated) exit 1 } +' <<<"$dry_run_section"; then + echo "e2e-dry-run should validate domain outputs before building" >&2 + exit 1 +fi + if ! grep -Fq "steps.e2e_domains.outputs.dry_packages" <<<"$dry_run_section"; then echo "e2e-dry-run should use resolved dry_packages instead of always running the full suite" exit 1 @@ -244,21 +379,21 @@ if ! grep -Fq "No dry-run CLI E2E needed" <<<"$dry_run_section"; then exit 1 fi -if ! grep -Fq "name: Resolve CLI E2E domains" <<<"$section" || - ! grep -Fq "id: e2e_domains" <<<"$section" || - ! grep -Fq "run: node scripts/e2e_domains.js" <<<"$section"; then - echo "e2e-live should resolve changed-file CLI E2E domains before credentials and tests" +if grep -Fq "name: Resolve CLI E2E domains" <<<"$section" || + grep -Fq "run: node scripts/e2e_domains.js" <<<"$section"; then + echo "e2e-live should reuse e2e-dry-run outputs instead of resolving domains again" exit 1 fi -if ! grep -Fq "steps.e2e_domains.outputs.live_packages" <<<"$section"; then - echo "e2e-live should use resolved live_packages instead of always running the full suite" +if ! grep -Fq "E2E_LIVE_PACKAGES: \${{ needs.e2e-dry-run.outputs.live_packages }}" <<<"$section"; then + echo "e2e-live should reuse live_packages resolved by e2e-dry-run" exit 1 fi -if ! grep -Fq "E2E_REASON: \${{ steps.e2e_domains.outputs.reason }}" <<<"$section" || +if ! grep -Fq "E2E_MODE: \${{ needs.e2e-dry-run.outputs.mode }}" <<<"$section" || + ! grep -Fq "E2E_REASON: \${{ needs.e2e-dry-run.outputs.reason }}" <<<"$section" || ! grep -Fq 'echo "Live CLI E2E domains: $E2E_MODE ($E2E_REASON)"' <<<"$section"; then - echo "e2e-live should pass dynamic domain output through env before shell use" + echo "e2e-live should consume the exact mode and reason produced by e2e-dry-run" exit 1 fi @@ -272,16 +407,23 @@ if ! awk ' exit 1 fi -if ! awk ' - /^ - name: Build lark-cli/ { in_step = 1 } - in_step && /if: \$\{\{ steps\.e2e_domains\.outputs\.mode != '\''skip'\'' \}\}/ { found = 1 } - in_step && /^ - name:/ && !/Build lark-cli/ { in_step = 0 } - END { exit found ? 0 : 1 } -' <<<"$section"; then - echo "e2e-live should skip building lark-cli when domain mode is skip" +if grep -Fq "steps.e2e_domains.outputs" <<<"$section"; then + echo "e2e-live should not retain step-local domain outputs after adopting the dry-run job gate" exit 1 fi +for step_name in "Build lark-cli" "Prepare shared live E2E tenant token"; do + live_setup_step="$(awk -v name="$step_name" ' + $0 == " - name: " name { in_step = 1 } + in_step { print } + in_step && /^ - name:/ && $0 != " - name: " name { exit } + ' <<<"$section")" + if grep -Eq '^ if:' <<<"$live_setup_step"; then + echo "e2e-live $step_name should run unconditionally after the non-skip job gate" >&2 + exit 1 + fi +done + if ! grep -Fq "permissions:" <<<"$section" || ! grep -Fq "contents: read" <<<"$section" || ! grep -Fq "checks: write" <<<"$section"; then @@ -346,13 +488,13 @@ fi if ! awk ' /^ - name: Prepare shared live E2E tenant token/ { in_step = 1 } in_step && /id: live_e2e_tat/ { has_id = 1 } - in_step && /if: \$\{\{ steps\.e2e_domains\.outputs\.mode != '\''skip'\'' \}\}/ { has_if = 1 } + in_step && /^ if:/ { has_if = 1 } in_step && /LARKSUITE_CLI_APP_ID: \$\{\{ secrets\.TEST_BOT1_APP_ID \}\}/ { has_app_id = 1 } in_step && /secrets\.TEST_BOT1_APP_SECRET/ { has_bot_credential = 1 } in_step && /node scripts\/fetch_e2e_tat\.js/ { has_script = 1 } in_step && /GITHUB_ENV/ { uses_github_env = 1 } in_step && /^ - name:/ && !/Prepare shared live E2E tenant token/ { in_step = 0 } - END { exit has_id && has_if && has_app_id && has_bot_credential && has_script && !uses_github_env ? 0 : 1 } + END { exit has_id && !has_if && has_app_id && has_bot_credential && has_script && !uses_github_env ? 0 : 1 } ' <<<"$section"; then echo "e2e-live should pass only a private tenant token file path through step output" exit 1 @@ -379,13 +521,18 @@ if ! awk ' exit 1 fi +if grep -Fq 'if [ "$E2E_MODE" = "skip" ]' <<<"$section"; then + echo "e2e-live should not retain an unreachable step-level skip branch" + exit 1 +fi + if grep -Fq "steps.live_e2e_credentials.outputs.configured" <<<"$section"; then echo "e2e-live build, configure, test, and report steps should not be gated by a skip-state output" exit 1 fi -if ! grep -Fq "if: \${{ !cancelled() && steps.e2e_domains.outputs.mode != 'skip' }}" <<<"$section"; then - echo "e2e-live report step should run after attempted live tests unless the workflow is cancelled or domain mode is skip" +if ! grep -Fq "if: \${{ !cancelled() }}" <<<"$section"; then + echo "e2e-live report step should run after attempted live tests unless the workflow is cancelled" exit 1 fi @@ -407,7 +554,7 @@ if grep -Fq '${{ secrets.CODECOV_TOKEN }}' <<<"$coverage_step" && fi if grep -Fq '${{ secrets.' <<<"$section" && - ! grep -Fq "if: \${{ $fork_safe_guard }}" <<<"$section"; then + ! grep -Fq "$fork_safe_guard" <<<"$section"; then echo "live E2E secrets should be available on push and same-repository pull_request, but not fork pull_request" >&2 exit 1 fi