Infra: Enable Merge Queue with required CI gates (#3832)

* Infra: Require CI checks before auto-merge

* Infra: Group required CI contexts

* Infra: Add stable required CI gates

* Infra: Remove repeated workflow comments

* Infra: Enable GitHub Merge Queue

* Infra: Remove unsupported CodeQL queue gate

* Infra: Document required CI dependencies

Author: kevinjqliu

.asf.yaml

@@ -36,13 +36,44 @@ github:
   protected_branches:
     main:
       required_status_checks:
-        # strict means "Require branches to be up to date before merging".
-        strict: true
+        contexts:
+          # Repository policy and security checks.
+          - Analyze Actions
+          - Run zizmor 🌈
+          - asf-allowlist-check
+          - rat
+
+          # Documentation checks.
+          - docs
+          - markdown-link-check
+
+          # Python workflow gates.
+          - python-ci-required
+          - python-integration-required
 
       required_pull_request_reviews:
         required_approving_review_count: 1
 
       required_linear_history: true
+  rulesets:
+    - name: Merge Queue
+      target: branch
+      enforcement: active
+      conditions:
+        ref_name:
+          include:
+            - "~DEFAULT_BRANCH"
+          exclude: []
+      rules:
+        - type: merge_queue
+          parameters:
+            check_response_timeout_minutes: 90
+            grouping_strategy: ALLGREEN
+            max_entries_to_build: 1
+            max_entries_to_merge: 1
+            merge_method: SQUASH
+            min_entries_to_merge: 1
+            min_entries_to_merge_wait_minutes: 0
   pull_requests:
     # allow pull requests to merge automatically once all requirements are met
     allow_auto_merge: true

.github/workflows/asf-allowlist-check.yml

@@ -25,6 +25,7 @@ name: "ASF Allowlist Check"
 
 on:
   pull_request:
+  merge_group:
   push:
     branches:
       - main

.github/workflows/check-md-link.yml

@@ -27,9 +27,7 @@ on:
     branches:
     - 'main'
   pull_request:
-    paths:
-      - '.github/workflows/check-md-link.yml'
-      - 'mkdocs/**'
+  merge_group:
   workflow_dispatch:
 
 permissions:

.github/workflows/codeql.yml

@@ -24,6 +24,7 @@ on:
     branches: [ "main" ]
   pull_request:
     branches: [ "main" ]
+  merge_group:
   schedule:
     - cron: '16 4 * * 1'
 

.github/workflows/license_check.yml

@@ -18,7 +18,9 @@
 #
 
 name: "Run License Check"
-on: pull_request
+on:
+  pull_request:
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/python-ci-docs.yml

@@ -24,6 +24,7 @@ on:
     branches:
       - 'main'
   pull_request:
+  merge_group:
 
 permissions:
   contents: read

.github/workflows/python-ci.yml

@@ -24,19 +24,7 @@ on:
     branches:
     - 'main'
   pull_request:
-    paths:
-    - '**'                              # Include all files and directories in the repository by default.
-    - '!.github/workflows/**'           # Exclude all workflow files
-    - '.github/workflows/python-ci.yml' # except the current file.
-    - '!.github/ISSUE_TEMPLATE/**'      # Exclude files and directories that don't impact tests or code like templates, metadata, and documentation.
-    - '!.gitignore'
-    - '!.asf.yml'
-    - '!mkdocs/**'
-    - '!.gitattributes'
-    - '!README.md'
-    - '!CONTRIBUTING.md'
-    - '!LICENSE'
-    - '!NOTICE'
+  merge_group:
 
 permissions:
   contents: read
@@ -127,3 +115,17 @@ jobs:
       run: uv sync --directory . --only-group dev --no-install-project
     - name: Mirror wheel CIBW_TEST_COMMAND
       run: uv run --directory . pytest tests/avro/test_decoder.py
+
+  python-ci-required:
+    if: ${{ always() }}
+    needs: [lint-and-unit-test, windows-unit-test, cibw-dev-env-smoke-test]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Verify Python CI jobs succeeded
+      env:
+        RESULTS: ${{ join(needs.*.result, ' ') }}
+      run: |
+        read -ra results <<< "$RESULTS"
+        for result in "${results[@]}"; do
+          test "$result" = "success"
+        done

.github/workflows/python-integration.yml

@@ -24,13 +24,7 @@ on:
     branches:
     - 'main'
   pull_request:
-    paths:                                       # Only run integration tests when Python (or the code they exercise) changes.
-    - '**/*.py'
-    - 'pyproject.toml'
-    - 'uv.lock'
-    - 'Makefile'
-    - 'dev/**'                                   # docker-compose files and fixtures used by the integration suites.
-    - '.github/workflows/python-integration.yml' # this file itself.
+  merge_group:
 
 permissions:
   contents: read
@@ -183,3 +177,17 @@ jobs:
         merge-multiple: true
     - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it!
       run: COVERAGE_FAIL_UNDER=75 make coverage-report
+
+  python-integration-required:
+    if: ${{ always() }}
+    needs: [integration-test, integration-test-s3, integration-test-adls, integration-test-gcs, integration-coverage-report]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Verify Python integration jobs succeeded
+      env:
+        RESULTS: ${{ join(needs.*.result, ' ') }}
+      run: |
+        read -ra results <<< "$RESULTS"
+        for result in "${results[@]}"; do
+          test "$result" = "success"
+        done

.github/workflows/zizmor.yml

@@ -24,6 +24,7 @@ on:
     branches: ["main"]
   pull_request:
     branches: ["**"]
+  merge_group:
 
 permissions: {}
 

AGENTS.md

@@ -62,6 +62,13 @@ lives under `pyiceberg/`, organized by concern rather than by engine:
 - Use existing test fixtures when possible.
 - We have a strong bias towards integration testing over mocks. Mocks should be avoided whenever possible and should only be used if similar, existing tests are using mocks.
 
+### Required CI and Merge Queue
+
+- Keep the `required_status_checks.contexts` list in `.asf.yaml` synchronized whenever a required job/check is added, renamed, or removed. Each entry is a job/check context name, not a workflow filename.
+- Verify that every required context reports for both `pull_request` and `merge_group`. Its producer workflow must run on both events.
+- When a workflow uses an aggregate required job, such as `python-ci-required`, keep `if: always()` and its `needs` list in sync with every job whose result should block merging.
+- Do not use `pull_request` path filters in workflows that produce required contexts. A skipped workflow does not report its required context, which blocks pull requests and causes Merge Queue entries to time out.
+
 ## Commands
 
 - **Install / set up dev env:** `make install` (installs `uv`, syncs all extras, builds Cython, installs pre-commit hooks)