diff --git a/.agentic-ci/config.yml b/.agentic-ci/config.yml
index 964dfc04e2..9e42844040 100644
--- a/.agentic-ci/config.yml
+++ b/.agentic-ci/config.yml
@@ -4,21 +4,27 @@
# https://github.com/opendatahub-io/agentic-ci, docs/sandbox-profiles.md).
# autofix reads it from the base branch before the agent runs. An overlay
# may set toolchains, setup, validate, skips and env, and open only the
-# goproxy, npm and pypi egress presets; anything wider is configured
-# centrally in autofix.
+# goproxy, npm, pypi and crates egress presets; anything wider is
+# configured centrally in autofix.
#
# Toolchains come from the agentic-ci catalog at the versions pinned in
-# mise.toml (python reads .python-version). If an upstream sync bumps a
+# mise.toml (python reads .python-version, rust reads rust-toolchain.toml). If an upstream sync bumps a
# version in mise.toml, bump it here too: mise does not install it, so
# the validate steps fail until the pins match. Setup installs mise from
# npm, links the catalog toolchains into mise so it never downloads
-# them, and installs the mise npm/go tools through the presets. Rust and
-# container, cluster and VM tasks cannot run in the sandbox and are
-# declared as skips.
+# them, and installs the mise npm/go tools through the presets. It also
+# fetches the crates of every tracked Cargo.lock through the crates
+# preset, so the Rust checks build offline, and runs rust:lint once to
+# warm the build cache, so clippy after the agent only rebuilds what it
+# changed. mise leaves rust alone (MISE_DISABLE_TOOLS) and the catalog's
+# cargo comes first on PATH.
+# cargo-deny, nextest and the container, cluster and VM tasks cannot run
+# in the sandbox and are declared as skips.
sandbox:
toolchains:
python: auto # reads .python-version
+ rust: auto # reads rust-toolchain.toml
node: "24.15.0"
go: "1.26.7"
buf: "1.72.0"
@@ -28,6 +34,7 @@ sandbox:
- goproxy
- npm
- pypi
+ - crates
env:
MISE_YES: "1"
@@ -62,12 +69,52 @@ sandbox:
mise link "buf@$(buf --version)" "$(root buf)"
mise link "helm@$(helm version --template '{{.Version}}' | sed 's/^v//')" \
"$(dirname "$(command -v helm)")"
- mise install \
- npm:markdownlint-cli2 \
- go:github.com/golangci/golangci-lint/v2/cmd/golangci-lint \
- go:google.golang.org/protobuf/cmd/protoc-gen-go \
- go:google.golang.org/grpc/cmd/protoc-gen-go-grpc \
- go:golang.org/x/tools/cmd/goimports
+ # A few go module fetches can fail at once when the sandbox's DNS
+ # relay times out; mise install is idempotent, so retry it.
+ ok=""
+ for attempt in 1 2 3; do
+ if mise install \
+ npm:markdownlint-cli2 \
+ go:github.com/golangci/golangci-lint/v2/cmd/golangci-lint \
+ go:google.golang.org/protobuf/cmd/protoc-gen-go \
+ go:google.golang.org/grpc/cmd/protoc-gen-go-grpc \
+ go:golang.org/x/tools/cmd/goimports; then
+ ok=1
+ break
+ fi
+ echo "mise install failed (attempt $attempt)" >&2
+ sleep 15
+ done
+ test -n "$ok"
+
+ - name: rust-crates
+ timeout: 1800
+ run: |
+ set -euo pipefail
+ # mise.toml sets RUSTC_WRAPPER=sccache for every `mise run`, and
+ # sccache is a GitHub download no preset opens. A pass-through
+ # wrapper runs rustc directly, without a cache.
+ mkdir -p "$HOME/.local/bin"
+ printf '#!/bin/sh\nexec "$@"\n' > "$HOME/.local/bin/sccache"
+ chmod +x "$HOME/.local/bin/sccache"
+ git ls-files -z -- ':(glob)**/Cargo.lock' |
+ while IFS= read -r -d '' lockfile; do
+ cargo fetch --locked --manifest-path "${lockfile%Cargo.lock}Cargo.toml"
+ done
+
+ - name: rust-warm
+ timeout: 3600
+ run: |
+ set -uo pipefail
+ # Build once before the agent with the exact rust:lint commands, so
+ # the agent's own clippy runs and the rust-lint validate step only
+ # rebuild what changed. A lint failure on the base commit does not
+ # matter here: the build cache is what this step is for.
+ status=0
+ mise run rust:lint || status=$?
+ if [ "$status" -ne 0 ]; then
+ echo "rust:lint exited $status during the warm-up; continuing, the build cache is what this step is for"
+ fi
validate:
- {name: python-lint, kind: lint, run: mise run python:lint, timeout: 600}
@@ -77,13 +124,22 @@ sandbox:
- {name: helm-lint, kind: lint, run: mise run helm:lint, timeout: 600}
- {name: go-format, kind: lint, run: mise run go:format:check, timeout: 600}
- {name: go-lint, kind: lint, run: mise run go:lint, timeout: 900}
+ - {name: rust-format, kind: lint, run: mise run rust:format:check, timeout: 600}
+ - {name: rust-lint, kind: lint, run: mise run rust:lint, timeout: 3600}
skips:
- - match: "rust:*, cargo, clippy, nextest, cargo-deny"
- reason: "No Rust toolchain or C compiler in the sandbox, so Rust builds, tests, lints and dependency checks rely on CI"
+ - match: "rust:deny, cargo deny, test:rust, cargo nextest, rust:verify:*"
+ reason: "cargo-deny and nextest are not in the toolchain catalog, and the Rust tests and verify scripts are left to CI"
- match: "docker:*, gateway:*, sandbox:*, vm:*, package:*"
reason: "Image, VM and package builds need a container runtime or privileged features the sandbox blocks"
- match: "e2e:*, test:*, helm:test"
reason: "End-to-end and cluster tests need a container runtime, a running gateway or a live cluster"
- match: "license:check"
reason: "Fails on main today: midstream files such as .tekton/*.yaml have no SPDX header"
+
+ # Build output stays in the sandbox and is never downloaded back.
+ discard_before_download:
+ - target
+ - e2e/rust/target
+ - examples/governance-interceptor/target
+ - examples/supervisor-middleware-content-guard/target
diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md
index 450d23094d..9fdf315c3c 100644
--- a/.agents/skills/helm-dev-environment/SKILL.md
+++ b/.agents/skills/helm-dev-environment/SKILL.md
@@ -236,8 +236,9 @@ cd ../../..
KUBECONFIG=kubeconfig mise run helm:gateway:apply
```
-`values-gateway.yaml` creates a `Gateway` (listener on port 80, class `eg`) and
-`GRPCRoute` in the `openshell` namespace. The `high-availability` profile
+`values-gateway.yaml` creates the `openshell` `Gateway` (listener on port 80,
+class `eg`) and an `openshell` `GRPCRoute` in the `openshell` namespace. The
+`high-availability` profile
installs the Envoy Gateway Helm chart and layers both
`values-high-availability.yaml` and `values-gateway.yaml` onto the OpenShell
release.
diff --git a/.agents/skills/test-release-canary/SKILL.md b/.agents/skills/test-release-canary/SKILL.md
index 1f6965873c..ed5491e87f 100644
--- a/.agents/skills/test-release-canary/SKILL.md
+++ b/.agents/skills/test-release-canary/SKILL.md
@@ -143,7 +143,7 @@ Loopback registration auto-derives the gateway name to `openshell` if `--name` i
| `macos`/`ubuntu-deb`/`fedora` job fails on `install.sh` | Dev release missing an asset, checksum mismatch, or `install.sh` regression on this branch. | Job log around the `curl … install.sh \| sh` step. |
| Sandbox create or exec fails | Published sandbox and supervisor artifacts are missing, incompatible, or cannot establish the protected runtime channel. | Gateway logs plus Docker, Podman, VM, Snap, or Kubernetes runtime diagnostics for the job. |
| `macos`/`ubuntu-deb`/`fedora` job fails on `openshell status` | Local gateway service did not start (systemd/brew/podman). Often a driver issue. | Service logs in the job log; `OPENSHELL_COMPUTE_DRIVER` env in the "Ensure …" step. |
-| `ubuntu-snap-system-docker` fails during `install.sh` | System Docker was unavailable, the edge revision or automatic interfaces were unavailable, or the gateway did not become reachable. | Failure diagnostics dump system Docker, snap service/connection/change state, gateway and snapd journals, snap logs, and port 17670 listeners. |
+| `ubuntu-snap-system-docker` fails during `install.sh` | System Docker was unavailable to the runner user, the edge revision or automatic interfaces were unavailable, or the user gateway did not become reachable. | Failure diagnostics dump system Docker, snap service/connection/change state, user and legacy gateway journals, snap logs, and port 17670 listeners. |
| `ubuntu-snap-system-docker` fails during the prover checks | The prover artifact is missing or packaged for the wrong architecture, `openshell.prover` is not exposed or confined to read the test policies, or its solver linkage is not runnable. | The `Verify Snap installation` and `Check a policy boundary with the Snap prover` steps, plus `snap info openshell` and `snap connections openshell`. |
| `ubuntu-snap-docker-preflight` unexpectedly succeeds | The installer no longer fails before installing the OpenShell snap when Docker is absent or supplied by the Docker snap. | Inspect `install.log`, `docker-snap.log`, `snap list`, and snapd changes. |
| `kubernetes` job fails on `helm install --wait` | Chart did not deploy in 5 min — usually image pull failure or readiness probe failing. | "Diagnostics on failure" step dumps `helm status`, manifest, pod describe, pod logs. |
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 4d32b414c1..14f2e5ef61 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -1,5 +1,35 @@
-# Broad ownership — core team reviews everything
+## Broad ownership — core team reviews everything
* @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
-# Vouch list — maintainers only (bot commits bypass, but manual edits need review)
+## Vouch list — maintainers only (bot commits bypass, but manual edits need review)
.github/VOUCHED.td @NVIDIA/openshell-codeowners
+
+## Merge enforcement — these paths decide who can merge, so they notify the full
+# list of maintainers to raise scrutiny on changes here.
+#
+# PR checks run local to the branch, so unless the check specifically avoids
+# using branch-local files, a PR could change its own gates and appear to pass
+# them. Covering the required checks here closes that gap.
+#
+# The whole workflows directory is covered, not just the approval gate, because
+# any workflow can claim a required check context or ask for a broader token.
+#
+# Normally, we'd use a GitHub team to cover all maintainers, but org-level
+# policies prevent outside collaborators from being on a GH team at this time
+# so we spell them out explicitly.
+
+# Core check mechanisms
+/.github/workflows/ @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/.github/actions/ @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/.github/CODEOWNERS @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/.github/zizmor.yml @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+
+# `MAINTAINERS.md` maintainer approval checks
+/MAINTAINERS.md @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/tasks/scripts/check_maintainer_approval.py @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/tasks/scripts/alert_maintainer_change.py @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/tasks/scripts/check_maintainer_approval_test.py @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+/tasks/scripts/alert_maintainer_change_test.py @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
+
+# Auditable SBOM check
+/tasks/scripts/verify-image-sbom.sh @NVIDIA/openshell-codeowners @mrunalp @sjenning @derekwaynecarr
diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml
index 89cc0ca920..4be954eced 100644
--- a/.github/workflows/integration-runner.yml
+++ b/.github/workflows/integration-runner.yml
@@ -89,3 +89,12 @@ jobs:
# Retry dependency preparation, then execute the test suite once.
nix build --no-link .#tmachine || nix build --no-link .#tmachine
nix run .#tmachine -- test "${ENVIRONMENT}" "${INSTALLER}" "${TESTSUITE}"
+
+ - name: Upload tmachine diagnostics
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: tmachine-diagnostics-${{ inputs.category }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ matrix.testsuite }}
+ path: artifacts/tmachine-diagnostics
+ if-no-files-found: ignore
+ retention-days: 7
diff --git a/.github/workflows/maintainer-approval.yml b/.github/workflows/maintainer-approval.yml
new file mode 100644
index 0000000000..f478ed97ac
--- /dev/null
+++ b/.github/workflows/maintainer-approval.yml
@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Maintainer Approval
+
+on:
+ merge_group:
+ types: [checks_requested]
+ pull_request_review:
+ types: [submitted, dismissed]
+
+permissions:
+ contents: read
+ pull-requests: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ maintainer-approval:
+ # This name is the required status check context. Changing it silently
+ # breaks the ruleset entry that gates merges on this job.
+ name: OpenShell / Maintainer Approval
+ if: github.repository_owner == 'NVIDIA'
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ steps:
+ # Check out the default branch, never the pull request head. Both the
+ # maintainer list and the decision helper must come from main: reading
+ # either from the pull request ref would let a contributor add themselves
+ # to the list, or rewrite the decision logic, and self-approve.
+ - name: Check out the maintainer list and helper
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: main
+ sparse-checkout: |
+ MAINTAINERS.md
+ tasks/scripts/check_maintainer_approval.py
+ sparse-checkout-cone-mode: false
+ persist-credentials: false
+
+ - name: Require an approving review from a maintainer
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ # merge_group carries no pull request in its payload, but the queue
+ # branch names the entry: gh-readonly-queue/main/pr-3027-.
+ MERGE_GROUP_REF: ${{ github.ref_name }}
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ if [ -z "${PR_NUMBER:-}" ]; then
+ if [[ "$MERGE_GROUP_REF" =~ /pr-([0-9]+)-[0-9a-f]+$ ]]; then
+ PR_NUMBER="${BASH_REMATCH[1]}"
+ else
+ # Fail closed: an unrecognised ref must never satisfy the gate.
+ echo "::error::No pull request resolved from '$MERGE_GROUP_REF'."
+ exit 1
+ fi
+ fi
+
+ gh api --paginate "repos/$GH_REPO/pulls/$PR_NUMBER/reviews" --jq '.[]' \
+ | jq -s '.' > reviews.json
+
+ # Exits non-zero when no maintainer's latest decisive review is an
+ # approval, and when MAINTAINERS.md yields no handles. That exit code
+ # is the check result; nothing is posted anywhere.
+ python3 tasks/scripts/check_maintainer_approval.py \
+ --maintainers MAINTAINERS.md \
+ --reviews reviews.json
diff --git a/.github/workflows/maintainers-change-alert.yml b/.github/workflows/maintainers-change-alert.yml
new file mode 100644
index 0000000000..d04052d51b
--- /dev/null
+++ b/.github/workflows/maintainers-change-alert.yml
@@ -0,0 +1,107 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: Maintainers Change Alert
+
+on:
+ pull_request_target:
+ types: [opened, reopened, synchronize]
+ paths:
+ - MAINTAINERS.md
+
+permissions:
+ contents: read
+ pull-requests: write
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+ cancel-in-progress: true
+
+jobs:
+ describe-change:
+ name: Comment on the approver set change if MAINTAINERS.md has changed
+ if: github.repository_owner == 'NVIDIA'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ # Default branch only. The helper must be the reviewed version, not
+ # whatever the pull request happens to contain.
+ - name: Check out the change-alert helper
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: main
+ sparse-checkout: tasks/scripts/alert_maintainer_change.py
+ sparse-checkout-cone-mode: false
+ persist-credentials: false
+
+ - name: Post the maintainer delta
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+ PR_NUMBER: ${{ github.event.pull_request.number }}
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ # Single source of truth for the comment marker: the script emits
+ # it as the first line of the body, the lookup below matches on it.
+ COMMENT_MARKER: ""
+ shell: bash
+ run: |
+ set -euo pipefail
+
+ # Fetching file contents is reading data, not executing it. The head
+ # revision is never checked out or run.
+ #
+ # A 404 means the file genuinely does not exist at that revision — a
+ # pull request that adds or deletes MAINTAINERS.md — and yields an
+ # empty side of the comparison. Every other failure is fatal: an empty
+ # file from a rate limit or a 5xx would render as "every maintainer
+ # was just added" or "nothing changed", both of which mislead the
+ # reviewer about who can merge code.
+ fetch_maintainers() {
+ local ref="$1" out="$2" err
+ err="$(mktemp)"
+ if gh api -H "Accept: application/vnd.github.raw" \
+ "repos/$GH_REPO/contents/MAINTAINERS.md?ref=$ref" > "$out" 2>"$err"; then
+ rm -f "$err"
+ return 0
+ fi
+ if grep -q 'HTTP 404' "$err"; then
+ rm -f "$err"
+ : > "$out"
+ return 0
+ fi
+ echo "::error::Could not fetch MAINTAINERS.md at $ref"
+ cat "$err" >&2
+ rm -f "$err"
+ return 1
+ }
+
+ fetch_maintainers "$BASE_SHA" before.md
+ fetch_maintainers "$HEAD_SHA" after.md
+
+ # An unparseable result exits non-zero, but the comment explaining
+ # why still has to be posted before this job fails.
+ status=0
+ python3 tasks/scripts/alert_maintainer_change.py \
+ --before before.md --after after.md > body.md || status=$?
+
+ # No output means the approver set did not change. A comment saying
+ # so is noise, so post nothing.
+ if [ -s body.md ]; then
+ cat body.md >> "$GITHUB_STEP_SUMMARY"
+
+ # Update the existing comment rather than stacking one per push.
+ COMMENT_ID=$(gh api --paginate "repos/$GH_REPO/issues/$PR_NUMBER/comments" \
+ --jq '.[] | select(.body | startswith($ENV.COMMENT_MARKER)) | .id' \
+ | head -n 1)
+
+ if [ -n "$COMMENT_ID" ]; then
+ gh api --method PATCH "repos/$GH_REPO/issues/comments/$COMMENT_ID" \
+ -F "body=@body.md" >/dev/null
+ else
+ gh api --method POST "repos/$GH_REPO/issues/$PR_NUMBER/comments" \
+ -F "body=@body.md" >/dev/null
+ fi
+ fi
+
+ exit "$status"
diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml
index 286d685672..a989d10e84 100644
--- a/.github/workflows/release-canary.yml
+++ b/.github/workflows/release-canary.yml
@@ -221,9 +221,10 @@ jobs:
- name: Install and check status
run: |
set -euo pipefail
- sudo systemctl set-environment \
+ systemctl --user set-environment \
"OPENSHELL_TELEMETRY_ENABLED=${OPENSHELL_TELEMETRY_ENABLED}"
- curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh
+ curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh |
+ DOCKER_HOST=unix:///tmp/nonexistent-docker.sock DOCKER_CONTEXT=nonexistent sh
sudo snap list openshell
if sudo snap list docker >/dev/null 2>&1; then
echo "install.sh unexpectedly installed the Docker snap" >&2
@@ -234,8 +235,12 @@ jobs:
sudo snap connections openshell | grep -E '^docker +openshell:docker +:docker +'
openshell --version
openshell.prover --version
- sudo snap services openshell
- sudo journalctl -b -u snap.openshell.gateway.service --no-pager |
+ snap services openshell
+ systemctl --user is-enabled --quiet snap.openshell.user-gateway.service
+ systemctl --user is-active snap.openshell.user-gateway.service
+ ! systemctl is-enabled --quiet snap.openshell.system-gateway.service
+ ! systemctl is-active --quiet snap.openshell.system-gateway.service
+ journalctl --user -u snap.openshell.user-gateway.service --no-pager |
grep -F "mTLS user authentication enabled"
openshell gateway list | grep -F "https://127.0.0.1:17670"
openshell status
@@ -280,10 +285,12 @@ jobs:
sudo snap services openshell
sudo snap connections openshell
sudo snap changes
- sudo systemctl status snap.openshell.gateway.service --no-pager
- sudo journalctl -b -u snap.openshell.gateway.service --no-pager -n 300
+ systemctl --user status snap.openshell.user-gateway.service --no-pager
+ journalctl --user -u snap.openshell.user-gateway.service --no-pager -n 300
+ sudo systemctl status snap.openshell.system-gateway.service --no-pager
+ sudo journalctl -b -u snap.openshell.system-gateway.service --no-pager -n 300
sudo journalctl -b -u snapd.service --no-pager -n 300
- sudo snap logs openshell.gateway -n=300
+ snap logs openshell.user-gateway -n=300
sudo ss -ltnp '( sport = :17670 )'
ubuntu-snap-docker-preflight:
diff --git a/.github/workflows/required-ci-gates.yml b/.github/workflows/required-ci-gates.yml
index 2310d27efe..aee374585d 100644
--- a/.github/workflows/required-ci-gates.yml
+++ b/.github/workflows/required-ci-gates.yml
@@ -285,5 +285,8 @@ jobs:
if [ "$CONTEXT_KIND" != "merge_group" ]; then
evaluate_workflow "OpenShell / E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e" "Core E2E result"
evaluate_workflow "OpenShell / GPU E2E" "branch-e2e.yml" "Branch E2E Checks" "test:e2e-gpu" "GPU E2E result"
+ else
+ post_status "OpenShell / E2E" success "Not applicable: E2E runs only on PRs" "$STATUS_TARGET_URL"
+ post_status "OpenShell / GPU E2E" success "Not applicable: GPU E2E runs only on PRs" "$STATUS_TARGET_URL"
fi
post_status "OpenShell / Helm Lint" success "Disabled in downstream fork (job requires runners unavailable downstream)" "$STATUS_TARGET_URL"
diff --git a/.github/zizmor.yml b/.github/zizmor.yml
index 7b1e035b77..27a89f878e 100644
--- a/.github/zizmor.yml
+++ b/.github/zizmor.yml
@@ -8,6 +8,7 @@ rules:
# head code. Keep each suppression scoped to its reviewed trigger block.
- dco.yml:3
- e2e-label-help.yml:13
+ - maintainers-change-alert.yml:6
- release-canary.yml:3
- required-ci-gates.yml:3
- vouch-check.yml:3
diff --git a/.tekton/odh-openshell-cli-pull-request.yaml b/.tekton/odh-openshell-cli-pull-request.yaml
index 0f0d68c786..41a6f2ba51 100644
--- a/.tekton/odh-openshell-cli-pull-request.yaml
+++ b/.tekton/odh-openshell-cli-pull-request.yaml
@@ -7,12 +7,14 @@ metadata:
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
pipelinesascode.tekton.dev/cancel-in-progress: "true"
pipelinesascode.tekton.dev/max-keep-runs: "3"
- pipelinesascode.tekton.dev/on-event: '[pull_request]'
pipelinesascode.tekton.dev/on-comment: '^/build-konflux'
- pipelinesascode.tekton.dev/on-target-branch: '[{{target_branch}}]'
+ build.appstudio.redhat.com/pull_request_number: "{{pull_request_number}}"
+ pipelinesascode.tekton.dev/on-cel-expression: |-
+ event == "pull_request"
+ && (files.all.exists(p, ! p.matches('^\\.tekton/')) || ".tekton/odh-openshell-cli-pull-request.yaml".pathChanged())
labels:
- appstudio.openshift.io/application: automation
- appstudio.openshift.io/component: pull-request-pipelines-odh-openshell-cli
+ appstudio.openshift.io/application: rhoai-v3-6
+ appstudio.openshift.io/component: odh-openshell-cli-v3-6
pipelines.appstudio.openshift.io/type: build
name: odh-openshell-cli-on-pull-request-{{pull_request_number}}
namespace: rhoai-tenant
@@ -23,31 +25,44 @@ spec:
- name: revision
value: '{{revision}}'
- name: output-image
- value: quay.io/rhoai/pull-request-pipelines:odh-openshell-cli-{{revision}}
+ value: quay.io/rhoai/odh-openshell-cli-rhel9:on-pr-{{pull_request_number}}
- name: dockerfile
value: deploy/docker/Dockerfile.konflux.cli
- name: path-context
value: .
- name: image-expires-after
- value: 5d
+ value: 30d
- name: enable-slack-failure-notification
value: "false"
-
- name: build-platforms
value:
- linux/x86_64
- linux-m2xlarge/arm64
+ - name: rhoai-version
+ value: "3.6.0"
+ - name: hermetic
+ value: true
+ - name: fips-check-blocking ##OCPEXCEPT-75 || OCPEXCEPT-76 || https://redhat-internal.slack.com/archives/C0C3BM63VFW/p1790927698394039?thread_ts=1790278918.493159&cid=C0C3BM63VFW
+ value: false
+ - name: prefetch-input
+ value: |
+ [
+ {"type": "cargo", "path": "."},
+ {"type": "rpm", "path": "deploy/konflux/cli"},
+ {"type": "generic", "path": "deploy/konflux/cli", "lockfile": "generic-fetcher.yaml"}
+ ]
+
pipelineRef:
params:
- name: url
value: https://github.com/red-hat-data-services/konflux-central.git
- name: revision
- value: '{{ target_branch }}'
+ value: '{{ cel: pac.target_branch.matches("^rhoai-\\d+\\.\\d+(-ea\\.\\d+)?$") ? pac.target_branch : "main" }}'
- name: pathInRepo
value: pipelines/multi-arch-container-build.yaml
resolver: git
taskRunTemplate:
- serviceAccountName: build-pipeline-pull-request-pipelines
+ serviceAccountName: build-pipeline-odh-openshell-cli-v3-6
podTemplate:
imagePullSecrets:
- name: redhat-appstudio-staginguser-pull-secret
diff --git a/.tekton/odh-openshell-e2e-pull-request.yaml b/.tekton/odh-openshell-e2e-pull-request.yaml
index 7b1f6a586e..fda2b683ab 100644
--- a/.tekton/odh-openshell-e2e-pull-request.yaml
+++ b/.tekton/odh-openshell-e2e-pull-request.yaml
@@ -39,6 +39,7 @@ spec:
[
{"type": "cargo", "path": "."},
{"type": "cargo", "path": "e2e/rust"},
+ {"type": "pip", "path": "odh", "requirements_files": ["requirements.txt"], "binary": {"packages": ":all:", "arch": "x86_64,aarch64", "os": "linux", "py_version": 311, "py_impl": "cp"}},
{"type": "rpm", "path": "deploy/konflux/e2e-odh"},
{"type": "generic", "path": "deploy/konflux/e2e-odh", "lockfile": "generic-fetcher.yaml"}
]
diff --git a/.tekton/odh-openshell-e2e-push.yaml b/.tekton/odh-openshell-e2e-push.yaml
index 3bcfb7c17f..13bfb21c07 100644
--- a/.tekton/odh-openshell-e2e-push.yaml
+++ b/.tekton/odh-openshell-e2e-push.yaml
@@ -40,6 +40,7 @@ spec:
[
{"type": "cargo", "path": "."},
{"type": "cargo", "path": "e2e/rust"},
+ {"type": "pip", "path": "odh", "requirements_files": ["requirements.txt"], "binary": {"packages": ":all:", "arch": "x86_64,aarch64", "os": "linux", "py_version": 311, "py_impl": "cp"}},
{"type": "rpm", "path": "deploy/konflux/e2e-odh"},
{"type": "generic", "path": "deploy/konflux/e2e-odh", "lockfile": "generic-fetcher.yaml"}
]
diff --git a/.tekton/odh-openshell-gateway-pull-request.yaml b/.tekton/odh-openshell-gateway-pull-request.yaml
index 15cacaa376..61e535c5d5 100644
--- a/.tekton/odh-openshell-gateway-pull-request.yaml
+++ b/.tekton/odh-openshell-gateway-pull-request.yaml
@@ -7,12 +7,14 @@ metadata:
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
pipelinesascode.tekton.dev/cancel-in-progress: "true"
pipelinesascode.tekton.dev/max-keep-runs: "3"
- pipelinesascode.tekton.dev/on-event: '[pull_request]'
pipelinesascode.tekton.dev/on-comment: '^/build-konflux'
- pipelinesascode.tekton.dev/on-target-branch: '[{{target_branch}}]'
+ build.appstudio.redhat.com/pull_request_number: "{{pull_request_number}}"
+ pipelinesascode.tekton.dev/on-cel-expression: |-
+ event == "pull_request"
+ && (files.all.exists(p, ! p.matches('^\\.tekton/')) || ".tekton/odh-openshell-gateway-pull-request.yaml".pathChanged())
labels:
- appstudio.openshift.io/application: automation
- appstudio.openshift.io/component: pull-request-pipelines-odh-openshell-gateway
+ appstudio.openshift.io/application: rhoai-v3-6
+ appstudio.openshift.io/component: odh-openshell-gateway-v3-6
pipelines.appstudio.openshift.io/type: build
name: odh-openshell-gateway-on-pull-request-{{pull_request_number}}
namespace: rhoai-tenant
@@ -23,31 +25,44 @@ spec:
- name: revision
value: '{{revision}}'
- name: output-image
- value: quay.io/rhoai/pull-request-pipelines:odh-openshell-gateway-{{revision}}
+ value: quay.io/rhoai/odh-openshell-gateway-rhel9:on-pr-{{pull_request_number}}
- name: dockerfile
value: deploy/docker/Dockerfile.konflux.gateway
- name: path-context
value: .
- name: image-expires-after
- value: 5d
+ value: 30d
- name: enable-slack-failure-notification
value: "false"
-
- name: build-platforms
value:
- linux/x86_64
- linux-m2xlarge/arm64
+ - name: rhoai-version
+ value: "3.6.0"
+ - name: hermetic
+ value: true
+ - name: fips-check-blocking ##OCPEXCEPT-75 || OCPEXCEPT-76 || https://redhat-internal.slack.com/archives/C0C3BM63VFW/p1790927698394039?thread_ts=1790278918.493159&cid=C0C3BM63VFW
+ value: false
+ - name: prefetch-input
+ value: |
+ [
+ {"type": "cargo", "path": "."},
+ {"type": "rpm", "path": "deploy/konflux/gateway"},
+ {"type": "generic", "path": "deploy/konflux/gateway", "lockfile": "generic-fetcher.yaml"}
+ ]
+
pipelineRef:
params:
- name: url
value: https://github.com/red-hat-data-services/konflux-central.git
- name: revision
- value: '{{ target_branch }}'
+ value: '{{ cel: pac.target_branch.matches("^rhoai-\\d+\\.\\d+(-ea\\.\\d+)?$") ? pac.target_branch : "main" }}'
- name: pathInRepo
value: pipelines/multi-arch-container-build.yaml
resolver: git
taskRunTemplate:
- serviceAccountName: build-pipeline-pull-request-pipelines
+ serviceAccountName: build-pipeline-odh-openshell-gateway-v3-6
podTemplate:
imagePullSecrets:
- name: redhat-appstudio-staginguser-pull-secret
diff --git a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml
new file mode 100644
index 0000000000..9144bb0ad9
--- /dev/null
+++ b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml
@@ -0,0 +1,60 @@
+apiVersion: tekton.dev/v1
+kind: PipelineRun
+#
+metadata:
+ annotations:
+ build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/openshell?rev={{revision}}
+ build.appstudio.redhat.com/commit_sha: '{{revision}}'
+ build.appstudio.redhat.com/target_branch: '{{target_branch}}'
+ pipelinesascode.tekton.dev/cancel-in-progress: "false"
+ pipelinesascode.tekton.dev/max-keep-runs: "3"
+ pipelinesascode.tekton.dev/on-cel-expression: |
+ event == "push"
+ && target_branch.matches("^refs/tags/v[0-9]+\\.[0-9]+\\.[0-9]+-rhaiv\\.[0-9]+$")
+ creationTimestamp: null
+ labels:
+ appstudio.openshift.io/application: opendatahub-builds
+ appstudio.openshift.io/component: odh-openshell-sandbox-openclaw-ci
+ pipelines.appstudio.openshift.io/type: build
+ name: odh-openshell-sandbox-openclaw-on-tag
+ namespace: open-data-hub-tenant
+spec:
+ params:
+ - name: git-url
+ value: '{{source_url}}'
+ - name: revision
+ value: '{{revision}}'
+ - name: output-image
+ value: quay.io/opendatahub/odh-openshell-sandbox-openclaw:{{git_tag}}
+ - name: dockerfile
+ value: deploy/docker/Dockerfile.konflux.openclaw
+ - name: path-context
+ value: .
+ - name: hermetic
+ value: "true"
+ - name: build-platforms
+ value:
+ - linux/x86_64
+ - linux/arm64
+ - name: prefetch-input
+ value: |
+ [
+ {"type": "npm", "path": "deploy/konflux/openclaw"},
+ {"type": "rpm", "path": "deploy/konflux/openclaw"}
+ ]
+ pipelineRef:
+ resolver: git
+ params:
+ - name: url
+ value: https://github.com/opendatahub-io/odh-konflux-central.git
+ - name: revision
+ value: main
+ - name: pathInRepo
+ value: pipeline/multi-arch-container-build.yaml
+ taskRunTemplate:
+ serviceAccountName: build-pipeline-odh-openshell-sandbox-openclaw-ci
+ workspaces:
+ - name: git-auth
+ secret:
+ secretName: '{{ git_auth_secret }}'
+status: {}
diff --git a/.tekton/odh-openshell-sandbox-pull-request.yaml b/.tekton/odh-openshell-sandbox-pull-request.yaml
index 0508c9f5ab..09e5894a7c 100644
--- a/.tekton/odh-openshell-sandbox-pull-request.yaml
+++ b/.tekton/odh-openshell-sandbox-pull-request.yaml
@@ -7,12 +7,14 @@ metadata:
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
pipelinesascode.tekton.dev/cancel-in-progress: "true"
pipelinesascode.tekton.dev/max-keep-runs: "3"
- pipelinesascode.tekton.dev/on-event: '[pull_request]'
pipelinesascode.tekton.dev/on-comment: '^/build-konflux'
- pipelinesascode.tekton.dev/on-target-branch: '[{{target_branch}}]'
+ build.appstudio.redhat.com/pull_request_number: "{{pull_request_number}}"
+ pipelinesascode.tekton.dev/on-cel-expression: |-
+ event == "pull_request"
+ && (files.all.exists(p, ! p.matches('^\\.tekton/')) || ".tekton/odh-openshell-sandbox-pull-request.yaml".pathChanged())
labels:
- appstudio.openshift.io/application: automation
- appstudio.openshift.io/component: pull-request-pipelines-odh-openshell-sandbox
+ appstudio.openshift.io/application: rhoai-v3-6
+ appstudio.openshift.io/component: odh-openshell-sandbox-v3-6
pipelines.appstudio.openshift.io/type: build
name: odh-openshell-sandbox-on-pull-request-{{pull_request_number}}
namespace: rhoai-tenant
@@ -23,31 +25,43 @@ spec:
- name: revision
value: '{{revision}}'
- name: output-image
- value: quay.io/rhoai/pull-request-pipelines:odh-openshell-sandbox-{{revision}}
+ value: quay.io/rhoai/odh-openshell-sandbox-rhel9:on-pr-{{pull_request_number}}
- name: dockerfile
value: deploy/docker/Dockerfile.konflux.sandbox
- name: path-context
value: .
- name: image-expires-after
- value: 5d
+ value: 30d
- name: enable-slack-failure-notification
value: "false"
-
- name: build-platforms
value:
- linux/x86_64
- linux-m2xlarge/arm64
+ - name: rhoai-version
+ value: "3.6.0"
+ - name: hermetic
+ value: true
+ - name: fips-check-blocking ##OCPEXCEPT-75 || OCPEXCEPT-76 || https://redhat-internal.slack.com/archives/C0C3BM63VFW/p1790927698394039?thread_ts=1790278918.493159&cid=C0C3BM63VFW
+ value: false
+ - name: prefetch-input
+ value: |
+ [
+ {"type": "cargo", "path": "."},
+ {"type": "rpm", "path": "deploy/konflux/sandbox"},
+ {"type": "generic", "path": "deploy/konflux/sandbox", "lockfile": "generic-fetcher.yaml"}
+ ]
pipelineRef:
params:
- name: url
value: https://github.com/red-hat-data-services/konflux-central.git
- name: revision
- value: '{{ target_branch }}'
+ value: '{{ cel: pac.target_branch.matches("^rhoai-\\d+\\.\\d+(-ea\\.\\d+)?$") ? pac.target_branch : "main" }}'
- name: pathInRepo
value: pipelines/multi-arch-container-build.yaml
resolver: git
taskRunTemplate:
- serviceAccountName: build-pipeline-pull-request-pipelines
+ serviceAccountName: build-pipeline-odh-openshell-sandbox-v3-6
podTemplate:
imagePullSecrets:
- name: redhat-appstudio-staginguser-pull-secret
diff --git a/.tekton/odh-openshell-supervisor-pull-request.yaml b/.tekton/odh-openshell-supervisor-pull-request.yaml
index 5db9355965..02e1572164 100644
--- a/.tekton/odh-openshell-supervisor-pull-request.yaml
+++ b/.tekton/odh-openshell-supervisor-pull-request.yaml
@@ -7,12 +7,14 @@ metadata:
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
pipelinesascode.tekton.dev/cancel-in-progress: "true"
pipelinesascode.tekton.dev/max-keep-runs: "3"
- pipelinesascode.tekton.dev/on-event: '[pull_request]'
pipelinesascode.tekton.dev/on-comment: '^/build-konflux'
- pipelinesascode.tekton.dev/on-target-branch: '[{{target_branch}}]'
+ build.appstudio.redhat.com/pull_request_number: "{{pull_request_number}}"
+ pipelinesascode.tekton.dev/on-cel-expression: |-
+ event == "pull_request"
+ && (files.all.exists(p, ! p.matches('^\\.tekton/')) || ".tekton/odh-openshell-supervisor-pull-request.yaml".pathChanged())
labels:
- appstudio.openshift.io/application: automation
- appstudio.openshift.io/component: pull-request-pipelines-odh-openshell-supervisor
+ appstudio.openshift.io/application: rhoai-v3-6
+ appstudio.openshift.io/component: odh-openshell-supervisor-v3-6
pipelines.appstudio.openshift.io/type: build
name: odh-openshell-supervisor-on-pull-request-{{pull_request_number}}
namespace: rhoai-tenant
@@ -23,31 +25,44 @@ spec:
- name: revision
value: '{{revision}}'
- name: output-image
- value: quay.io/rhoai/pull-request-pipelines:odh-openshell-supervisor-{{revision}}
+ value: quay.io/rhoai/odh-openshell-supervisor-rhel9:on-pr-{{pull_request_number}}
- name: dockerfile
value: deploy/docker/Dockerfile.konflux.supervisor
- name: path-context
value: .
- name: image-expires-after
- value: 5d
+ value: 30d
- name: enable-slack-failure-notification
value: "false"
-
- name: build-platforms
value:
- linux/x86_64
- linux-m2xlarge/arm64
+ - name: rhoai-version
+ value: "3.6.0"
+ - name: hermetic
+ value: true
+ - name: fips-check-blocking ##OCPEXCEPT-75 || OCPEXCEPT-76 || https://redhat-internal.slack.com/archives/C0C3BM63VFW/p1790927698394039?thread_ts=1790278918.493159&cid=C0C3BM63VFW
+ value: false
+ - name: prefetch-input
+ value: |
+ [
+ {"type": "cargo", "path": "."},
+ {"type": "rpm", "path": "deploy/konflux/supervisor"},
+ {"type": "generic", "path": "deploy/konflux/supervisor", "lockfile": "generic-fetcher.yaml"}
+ ]
+
pipelineRef:
params:
- name: url
value: https://github.com/red-hat-data-services/konflux-central.git
- name: revision
- value: '{{ target_branch }}'
+ value: '{{ cel: pac.target_branch.matches("^rhoai-\\d+\\.\\d+(-ea\\.\\d+)?$") ? pac.target_branch : "main" }}'
- name: pathInRepo
value: pipelines/multi-arch-container-build.yaml
resolver: git
taskRunTemplate:
- serviceAccountName: build-pipeline-pull-request-pipelines
+ serviceAccountName: build-pipeline-odh-openshell-supervisor-v3-6
podTemplate:
imagePullSecrets:
- name: redhat-appstudio-staginguser-pull-secret
diff --git a/CI.md b/CI.md
index f38504aef3..de6c53511b 100644
--- a/CI.md
+++ b/CI.md
@@ -149,6 +149,12 @@ compatibility baseline for the v1beta1 Sandbox API. It does not track the local
K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through
its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version.
+The `ubuntu-k3s` lane registers the gateway's ClusterIP Service directly, so
+recreating the Service requires reprovisioning the fixture. Before conformance,
+every installer waits up to five minutes for a connected gateway; interactive
+shells skip the wait. K3s failures upload diagnostics as `tmachine-diagnostics-*`
+artifacts.
+
### Run only the policy advisor conformance tests
Manually dispatch `Integration Tests` on the candidate branch with an
@@ -487,7 +493,7 @@ its own stable result status.
Merge-group runs use the `merge_group` event. The event is distinct from `pull_request` and `push`, and GitHub will not report required checks for queued PRs unless the workflows include it. In this repository:
- `Branch Checks` runs the standard non-E2E gates on the merge-group SHA.
-- `Branch E2E Checks` does not run for merge groups. E2E suites remain opt-in on PRs.
+- `Branch E2E Checks` does not run for merge groups. E2E suites remain opt-in on PRs. `Required CI Gates` publishes successful not-applicable statuses for both E2E contexts on merge-group events and subsequent merge-group workflow completions.
- `Helm Lint` runs for merge groups without the PR diff optimization, because the merge-group branch is the final integration state.
- `Trivy Changes` compares the merge-group configuration with its base and rejects new High or Critical findings.
- `Required CI Gates` posts the same `OpenShell / ...` statuses to the merge-group SHA and does not require a `pull-request/` mirror for merge-group events.
@@ -543,7 +549,7 @@ These workflows run after merge to publish dev/tagged artifacts and verify them.
| `.github/workflows/release-tag.yml` | Publishes tagged stable releases and manually dispatched pre-releases. Its automatic tag trigger excludes `-pre.*`. Both require the currently implemented qualification profile to pass before publication; the summary identifies the remaining RFC 0014 coverage. Failed candidates retain build artifacts and evidence in Actions storage. Source-SHA OCI images remain available as qualification inputs. |
| `.github/workflows/snap-package.yml` | Builds Snap and component artifacts for Release Dev and Release Tag without Store credentials or publication. |
| `.github/workflows/snap-publish.yml` | Uploads existing Snap and component artifacts to the Store without rebuilding. Release Dev calls it directly for `latest/edge` after Snap builds and requires it to succeed before creating the dev release. Release Tag calls it for `latest/stable` only for stable releases, after qualification and release assembly succeed. |
-| `.github/workflows/release-canary.yml` | Smoke-tests published dev artifacts in the `macos`, `ubuntu-deb`, `ubuntu-snap-system-docker`, `fedora`, and `kubernetes` (kind + Helm) jobs. Each job reaches its gateway and creates, exercises, and deletes a sandbox. The Snap lanes verify a compatible system Docker lifecycle and `ubuntu-snap-docker-preflight` tests fail-fast behavior when Docker is absent or supplied by the Docker snap. The positive Snap lane also runs a local policy containment check with the packaged prover. It runs automatically after `Release Dev` succeeds and supports manual dispatch (`gh workflow run release-canary.yml --ref `). See the `test-release-canary` skill for the playbook and local kind reproduction. |
+| `.github/workflows/release-canary.yml` | Smoke-tests published dev artifacts in the `macos`, `ubuntu-deb`, `ubuntu-snap-system-docker`, `fedora`, and `kubernetes` (kind + Helm) jobs. Each job reaches its gateway and creates, exercises, and deletes a sandbox. The `ubuntu-snap-system-docker` job verifies a compatible system Docker lifecycle, while `ubuntu-snap-docker-preflight` tests fail-fast behavior when Docker is absent or supplied by the Docker snap. The positive Snap lane also runs a local policy containment check with the packaged prover. It runs automatically after `Release Dev` succeeds and supports manual dispatch (`gh workflow run release-canary.yml --ref `). See the `test-release-canary` skill for the playbook and local kind reproduction. |
## Required status contexts
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7ea44911a0..77bc795197 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -68,6 +68,12 @@ Do not start substantial issue-backed work until a maintainer has accepted the i
Use agents and the repository skills as needed to understand the affected code, evaluate tradeoffs, implement the smallest coherent change, and verify it. The pull request should explain what changed and how it was tested; it should not substitute an agent transcript for the contributor's understanding.
+Every pull request must be approved by someone listed in [MAINTAINERS.md](MAINTAINERS.md) before it can merge. This is enforced by the `OpenShell / Maintainer Approval` status check, which turns green once one of those reviewers approves. Reviews from other contributors are welcome and count toward the general approval requirement, but they do not satisfy this check.
+
+Pull requests that touch the enforcement machinery itself — anything under `.github/workflows/`, `.github/actions/`, `MAINTAINERS.md`, `.github/CODEOWNERS`, or the enforcement scripts in `tasks/scripts/` — additionally require a code owner's approval, listed in [.github/CODEOWNERS](.github/CODEOWNERS). The status check reads its inputs from `main`, but a workflow triggered by a review runs the pull request's own copy of the workflow file, so GitHub's native code owner requirement is what keeps a change from disabling the gate that would have blocked it.
+
+Maintainers are not requested automatically. If your pull request has been idle, ask for a reviewer in the pull request or in the CNCF Slack channel rather than waiting.
+
## Agent Skills
OpenShell keeps skills for using the product separate from skills for developing the repository.
diff --git a/Cargo.lock b/Cargo.lock
index e8db9e8103..35193cd8f4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3975,16 +3975,15 @@ dependencies = [
[[package]]
name = "noyalib"
-version = "0.0.28"
+version = "0.0.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f075ef19fa3bcf8697c0ef96c37d5c435d339a40ab8081cae3aac3a4e7fee9a"
+checksum = "962838abbd8dd2cd3844d2208dc502458bba6985f8bf1dc21761b30260fcbd7f"
dependencies = [
"hashbrown 0.17.0",
"indexmap",
"libm",
"memchr",
"rustc-hash 2.1.2",
- "serde",
"serde_core",
"smallvec",
]
diff --git a/Cargo.toml b/Cargo.toml
index 0882a15adb..c6136bdc26 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -86,7 +86,7 @@ socket2 = "0.6"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_path_to_error = "0.1"
-serde_yml = { package = "noyalib", version = "0.0.28", default-features = false, features = ["std", "compat-serde-yaml"] }
+serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] }
toml = "0.8"
apollo-parser = "0.8.5"
tower-mcp-types = "=0.22.2"
diff --git a/crates/openshell-cli/src/output.rs b/crates/openshell-cli/src/output.rs
index 7a8120bf3b..aee8a2c198 100644
--- a/crates/openshell-cli/src/output.rs
+++ b/crates/openshell-cli/src/output.rs
@@ -91,7 +91,10 @@ where
"yaml" => {
let envelope =
paginated_collection_value(collection_name, items, next_page_token, to_json);
- print!("{}", serde_yml::to_string(&envelope).into_diagnostic()?);
+ print!(
+ "{}",
+ openshell_core::yaml::to_string(&envelope).into_diagnostic()?
+ );
Ok(true)
}
"table" => Ok(false),
@@ -138,7 +141,10 @@ where
}
"yaml" => {
let values: Vec<_> = items.iter().map(to_json).collect();
- print!("{}", serde_yml::to_string(&values).into_diagnostic()?);
+ print!(
+ "{}",
+ openshell_core::yaml::to_string(&values).into_diagnostic()?
+ );
Ok(true)
}
"table" => Ok(false),
@@ -177,7 +183,10 @@ where
}
"yaml" => {
let value = to_json(item);
- print!("{}", serde_yml::to_string(&value).into_diagnostic()?);
+ print!(
+ "{}",
+ openshell_core::yaml::to_string(&value).into_diagnostic()?
+ );
Ok(true)
}
"table" => Ok(false),
@@ -265,7 +274,7 @@ where
write!(
writer,
"{}",
- serde_yml::to_string(&values).into_diagnostic()?
+ openshell_core::yaml::to_string(&values).into_diagnostic()?
)
.into_diagnostic()?;
Ok(true)
@@ -309,7 +318,7 @@ where
write!(
writer,
"{}",
- serde_yml::to_string(&envelope).into_diagnostic()?
+ openshell_core::yaml::to_string(&envelope).into_diagnostic()?
)
.into_diagnostic()?;
Ok(true)
@@ -356,7 +365,7 @@ where
write!(
writer,
"{}",
- serde_yml::to_string(&value).into_diagnostic()?
+ openshell_core::yaml::to_string(&value).into_diagnostic()?
)
.into_diagnostic()?;
Ok(true)
diff --git a/crates/openshell-cli/tests/ensure_providers_integration.rs b/crates/openshell-cli/tests/ensure_providers_integration.rs
index e7221e0079..c08f49296b 100644
--- a/crates/openshell-cli/tests/ensure_providers_integration.rs
+++ b/crates/openshell-cli/tests/ensure_providers_integration.rs
@@ -339,6 +339,13 @@ impl OpenShell for TestOpenShell {
}))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
request: tonic::Request,
diff --git a/crates/openshell-cli/tests/mtls_integration.rs b/crates/openshell-cli/tests/mtls_integration.rs
index ca969dde13..6a4bafd19d 100644
--- a/crates/openshell-cli/tests/mtls_integration.rs
+++ b/crates/openshell-cli/tests/mtls_integration.rs
@@ -295,6 +295,13 @@ impl OpenShell for TestOpenShell {
))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
_request: tonic::Request,
diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs
index 2cbeaffdf6..e53352d079 100644
--- a/crates/openshell-cli/tests/provider_commands_integration.rs
+++ b/crates/openshell-cli/tests/provider_commands_integration.rs
@@ -751,6 +751,13 @@ impl OpenShell for TestOpenShell {
}))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
request: tonic::Request,
diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs
index bb0abceb2f..f71108691e 100644
--- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs
+++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs
@@ -547,6 +547,13 @@ impl OpenShell for TestOpenShell {
Ok(Response::new(ProviderResponse::default()))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
_request: tonic::Request,
diff --git a/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs b/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs
index 359895838a..c57e2d4ea5 100644
--- a/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs
+++ b/crates/openshell-cli/tests/sandbox_exec_streaming_integration.rs
@@ -243,6 +243,7 @@ mock_gateway! {
delete_service(proto::DeleteServiceRequest) -> proto::DeleteServiceResponse;
revoke_ssh_session(proto::RevokeSshSessionRequest) -> proto::RevokeSshSessionResponse;
create_provider(proto::CreateProviderRequest) -> proto::ProviderResponse;
+ get_provider_credentials(proto::GetProviderCredentialsRequest) -> proto::GetProviderCredentialsResponse;
get_provider(proto::GetProviderRequest) -> proto::ProviderResponse;
list_providers(proto::ListProvidersRequest) -> proto::ListProvidersResponse;
list_provider_profiles(proto::ListProviderProfilesRequest) -> proto::ListProviderProfilesResponse;
diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs
index 7a5780d7f3..fb8d39895f 100644
--- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs
+++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs
@@ -357,6 +357,13 @@ impl OpenShell for TestOpenShell {
Ok(Response::new(ProviderResponse::default()))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
_request: tonic::Request,
diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs
index 56e314127d..12cf145f44 100644
--- a/crates/openshell-core/src/config.rs
+++ b/crates/openshell-core/src/config.rs
@@ -410,6 +410,10 @@ pub struct OidcConfig {
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MtlsAuthConfig {
+ /// Enable gateway-wide operators authenticated by verified certificates with OU=operator.
+ /// Independent of ordinary mTLS user authentication; defaults off.
+ #[serde(default)]
+ pub operator_enabled: bool,
/// When true, the gateway maps a verified TLS client certificate into a
/// user principal. Sandbox and supervisor clients use bearer identity, so
/// this setting is independent of the selected compute driver.
diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs
index 1cbf6968e0..84d4f8ccf5 100644
--- a/crates/openshell-core/src/lib.rs
+++ b/crates/openshell-core/src/lib.rs
@@ -42,6 +42,7 @@ pub mod net;
pub mod oauth;
pub mod paths;
pub mod policy;
+pub use openshell_policy_schema::yaml;
pub mod policy_identity;
pub mod progress;
pub mod proposals;
diff --git a/crates/openshell-gateway-interceptors/src/proto_json.rs b/crates/openshell-gateway-interceptors/src/proto_json.rs
index a67ae30c5c..f34c791adc 100644
--- a/crates/openshell-gateway-interceptors/src/proto_json.rs
+++ b/crates/openshell-gateway-interceptors/src/proto_json.rs
@@ -336,6 +336,28 @@ mod tests {
assert_eq!(decoded, request);
}
+ #[test]
+ fn operator_credential_response_omits_values_inside_maps() {
+ let codec = ProtoJsonCodec::openshell().unwrap();
+ let response = openshell_core::proto::GetProviderCredentialsResponse {
+ credentials: HashMap::from([(
+ "ACCESS_TOKEN".to_string(),
+ openshell_core::proto::ProviderCredentialValue {
+ value: "must-not-observe".to_string(),
+ expiration_time: None,
+ },
+ )]),
+ };
+ let view = codec
+ .decode_bytes_to_interceptor_json(
+ "openshell.v1.GetProviderCredentialsResponse",
+ &response.encode_to_vec(),
+ )
+ .unwrap();
+ assert!(view["credentials"]["ACCESS_TOKEN"].get("value").is_none());
+ assert!(!view.to_string().contains("must-not-observe"));
+ }
+
#[test]
fn interceptor_view_omits_nested_secrets_but_keeps_non_secret_fields() {
let codec = ProtoJsonCodec::openshell().unwrap();
diff --git a/crates/openshell-policy-schema/README.md b/crates/openshell-policy-schema/README.md
new file mode 100644
index 0000000000..7debf228b3
--- /dev/null
+++ b/crates/openshell-policy-schema/README.md
@@ -0,0 +1,16 @@
+# openshell-policy-schema
+
+Canonical authored policy types, bounded YAML parsing, and pure validation.
+Runtime enforcement and protobuf adaptation remain outside this crate.
+
+The `yaml` module supplies the shared authored-data compatibility boundary used
+by policy, provider-profile, and prover loaders and by CLI YAML output. Each
+loader declares the schema object paths that must reject null before ordinary
+Serde decoding. Optional objects and untyped user-data values retain their
+ordinary null semantics. Empty or null top-level documents are rejected.
+Loaders reject duplicate mapping keys before decoding their typed schema.
+
+Policy parsing retains its explicit resource budgets and field-path diagnostics.
+YAML exports quote strings and mapping keys that legacy readers could interpret
+as dates, numbers, or booleans, without changing actual numeric or boolean data.
+Formatting the already-generated output does not impose new input-size limits.
diff --git a/crates/openshell-policy-schema/src/lib.rs b/crates/openshell-policy-schema/src/lib.rs
index 8642ce8599..656433b794 100644
--- a/crates/openshell-policy-schema/src/lib.rs
+++ b/crates/openshell-policy-schema/src/lib.rs
@@ -17,6 +17,8 @@ use std::str::FromStr;
use miette::{IntoDiagnostic, Result, WrapErr};
use serde::{Deserialize, Deserializer, Serialize};
+pub mod yaml;
+
/// Fixed resource bound for legacy MCP request batches inspected by Tower.
pub const MAX_MCP_LEGACY_BATCH_MESSAGES: usize = 64;
@@ -518,7 +520,9 @@ where
D: Deserializer<'de>,
T: Deserialize<'de>,
{
- T::deserialize(deserializer).map(Some)
+ Option::::deserialize(deserializer)?
+ .map(Some)
+ .ok_or_else(|| serde::de::Error::custom("null is not allowed for this field"))
}
const MAX_UNKNOWN_FIELD_PATH_BYTES: usize = 1_024;
@@ -591,6 +595,36 @@ pub fn parse_policy_with_limits(source: &str, limits: ParseLimits) -> Result String {
/// Serialize the authored representation to YAML.
pub fn serialize_policy(document: &PolicyDocument) -> Result {
- serde_yml::to_string(document)
+ yaml::to_string(document)
.into_diagnostic()
.wrap_err("failed to serialize policy to YAML")
}
@@ -1198,6 +1232,55 @@ mod tests {
}
}
+ #[test]
+ fn nested_null_objects_remain_invalid() {
+ for spelling in ["null", "~", ""] {
+ for body in [
+ format!("network_policies: {spelling}"),
+ format!("network_policies: {{x: {spelling}}}"),
+ format!("network_middlewares: {spelling}"),
+ format!("landlock: {spelling}"),
+ format!("network_middlewares: {{x: {{middleware: foo, config: {spelling}}}}}"),
+ format!("network_middlewares: {{x: {{middleware: foo, endpoints: {spelling}}}}}"),
+ format!("network_policies: {{x: {{endpoints: [{spelling}]}}}}"),
+ ] {
+ // A bare [] is an empty collection rather than a null entry.
+ if spelling.is_empty() && body.ends_with("[]}}") {
+ continue;
+ }
+ assert!(
+ parse_policy(&format!("version: 1\n{body}\n")).is_err(),
+ "{body}"
+ );
+ }
+ for field in [
+ format!("rules: [{{allow: {spelling}}}]"),
+ format!("deny_rules: [{{method: GET, query: {spelling}}}]"),
+ format!("deny_rules: [{spelling}]"),
+ format!("rules: [{{allow: {{query: {spelling}}}}}]"),
+ format!("rules: [{{allow: {{params: {spelling}}}}}]"),
+ format!("rules: [{{allow: {{tool: {spelling}}}}}]"),
+ format!("deny_rules: [{{tool: {spelling}}}]"),
+ format!("graphql_persisted_queries: {spelling}"),
+ format!("graphql_persisted_queries: {{op: {spelling}}}"),
+ format!("credential_binding: {spelling}"),
+ format!("json_rpc: {spelling}"),
+ format!("mcp: {spelling}"),
+ ] {
+ if field == "deny_rules: []" {
+ continue;
+ }
+ let yaml = format!(
+ "version: 1\nnetwork_policies: {{x: {{endpoints: [{{host: example.com, port: 443, {field}}}]}}}}\n"
+ );
+ assert!(parse_policy(&yaml).is_err(), "{yaml}");
+ }
+ }
+ let yaml =
+ "version: 1\nnetwork_middlewares: {x: {middleware: foo, config: {optional: null}}}\n";
+ assert!(parse_policy(yaml).is_ok());
+ }
+
#[test]
fn truncation_paths_are_unicode_safe() {
let oversized = "é".repeat(ParseLimits::default().max_bytes);
diff --git a/crates/openshell-policy-schema/src/yaml.rs b/crates/openshell-policy-schema/src/yaml.rs
new file mode 100644
index 0000000000..3f3c072d9e
--- /dev/null
+++ b/crates/openshell-policy-schema/src/yaml.rs
@@ -0,0 +1,271 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! YAML compatibility at `OpenShell`'s authored-data boundaries.
+
+use serde::Serialize;
+use serde::de::DeserializeOwned;
+use serde_yml::{Error, Value};
+
+/// Load authored YAML with bounded parsing and duplicate-key rejection.
+pub fn from_str(source: &str) -> Result {
+ from_str_with_object_paths(source, &[])
+}
+
+/// Reject null at schema-declared object paths before ordinary typed decoding.
+///
+/// A `*` selects each sequence element or mapping value. Missing fields are
+/// left to Serde defaults; optional objects should not be listed themselves.
+pub fn from_str_with_object_paths(
+ source: &str,
+ paths: &[&str],
+) -> Result {
+ let config =
+ serde_yml::ParserConfig::new().duplicate_key_policy(serde_yml::DuplicateKeyPolicy::Error);
+ let value: Value = serde_yml::from_str_with_config(source, &config)?;
+ reject_null_objects(&value, paths)?;
+ T::deserialize(serde_yml::Deserializer::new(&value))
+}
+
+/// Validate only declared schema objects, not arbitrary user-data mappings.
+pub fn reject_null_objects(value: &Value, paths: &[&str]) -> Result<(), Error> {
+ reject_null_at_path(value, "", "")?;
+ for path in paths {
+ reject_null_at_path(value, path, "")?;
+ }
+ Ok(())
+}
+
+fn reject_null_at_path(value: &Value, remaining: &str, location: &str) -> Result<(), Error> {
+ if let Value::Tagged(tagged) = value {
+ return reject_null_at_path(tagged.value(), remaining, location);
+ }
+ if remaining.is_empty() {
+ return if value.is_null() {
+ Err(Error::Parse(format!(
+ "{}: null is not an object",
+ if location.is_empty() {
+ ".".to_owned()
+ } else {
+ crate::bound_path(location.to_owned())
+ }
+ )))
+ } else {
+ Ok(())
+ };
+ }
+ let (field, rest) = remaining.split_once('.').unwrap_or((remaining, ""));
+ if field == "*" {
+ if let Some(mapping) = value.as_mapping() {
+ for (key, child) in mapping {
+ reject_null_at_path(child, rest, &format!("{location}[{:?}]", key.as_str()))?;
+ }
+ } else if let Some(sequence) = value.as_sequence() {
+ for (index, child) in sequence.iter().enumerate() {
+ reject_null_at_path(child, rest, &format!("{location}[{index}]"))?;
+ }
+ }
+ } else if let Some(child) = value.as_mapping().and_then(|mapping| mapping.get(field)) {
+ let path = if location.is_empty() {
+ field.to_owned()
+ } else {
+ format!("{location}.{field}")
+ };
+ reject_null_at_path(child, rest, &path)?;
+ }
+ Ok(())
+}
+
+/// Preserve authored strings even for YAML 1.1 readers (dates, timestamps,
+/// underscore-separated integers, and legacy boolean words).
+pub fn to_string(value: &T) -> Result {
+ let value = serde_yml::to_value(&value)?;
+ let mut yaml = serde_yml::to_string(&value)?;
+ let mut strings = Vec::new();
+ collect_strings(&value, "", &mut strings);
+ if strings.is_empty() {
+ return Ok(yaml);
+ }
+ // This is our already-emitted output, not a fresh untrusted input. Size the
+ // formatting walk to that output so an otherwise valid large export does
+ // not acquire the input parser's collection or byte limits.
+ let mut config = serde_yml::ParserConfig::new();
+ config.max_document_length = config.max_document_length.max(yaml.len());
+ config.max_total_scalar_bytes = config.max_total_scalar_bytes.max(yaml.len());
+ config.max_events = config.max_events.max(yaml.len().saturating_mul(4));
+ config.max_nodes = config.max_nodes.max(yaml.len().saturating_mul(2));
+ config.max_mapping_keys = config.max_mapping_keys.max(yaml.len());
+ config.max_sequence_length = config.max_sequence_length.max(yaml.len());
+ let document = serde_yml::cst::parse_document_with_config(&yaml, &config)?;
+ let mut edits = Vec::new();
+ for (path, string, is_key) in strings {
+ let (start, end) = if is_key {
+ document.key_span(&path)
+ } else {
+ document.span_at(&path)
+ }
+ .ok_or_else(|| Error::Parse("serialized YAML string missing".into()))?;
+ // Already quoted or block-styled strings retain their emitter style.
+ if !yaml[start..end].starts_with(['\'', '"', '|', '>']) {
+ let quoted = serde_yml::to_string_with_config(
+ &string,
+ &serde_yml::SerializerConfig::new().quote_all(true),
+ )?;
+ edits.push((start, end, quoted.trim_end().to_owned()));
+ }
+ }
+ edits.sort_unstable_by_key(|(start, _, _)| *start);
+ for (start, end, quoted) in edits.into_iter().rev() {
+ yaml.replace_range(start..end, "ed);
+ }
+ Ok(yaml)
+}
+
+fn collect_strings<'a>(value: &'a Value, path: &str, result: &mut Vec<(String, &'a str, bool)>) {
+ match value {
+ Value::String(string) if needs_legacy_quotes(string) => {
+ result.push((path.into(), string, false));
+ }
+ Value::Mapping(mapping) => {
+ for (key, child) in mapping {
+ let child_path = format!("{path}{}", serde_yml::path::quote_key(key.as_str()));
+ if needs_legacy_quotes(key.as_str()) {
+ result.push((child_path.clone(), key.as_str(), true));
+ }
+ collect_strings(child, &child_path, result);
+ }
+ }
+ Value::Sequence(sequence) => {
+ for (index, child) in sequence.iter().enumerate() {
+ collect_strings(child, &format!("{path}[{index}]"), result);
+ }
+ }
+ Value::Tagged(tagged) => collect_strings(tagged.value(), path, result),
+ _ => {}
+ }
+}
+
+fn needs_legacy_quotes(string: &str) -> bool {
+ let unsigned = string.trim_start_matches(['+', '-']);
+ (unsigned.as_bytes().first().is_some_and(u8::is_ascii_digit)
+ && unsigned.bytes().all(|byte| {
+ byte.is_ascii_digit()
+ || matches!(
+ byte,
+ b'_' | b'.'
+ | b':'
+ | b'/'
+ | b'+'
+ | b'-'
+ | b' '
+ | b'T'
+ | b't'
+ | b'Z'
+ | b'z'
+ | b'E'
+ | b'e'
+ | b'X'
+ | b'x'
+ | b'O'
+ | b'o'
+ | b'B'
+ | b'b'
+ )
+ }))
+ || matches!(
+ string.to_ascii_lowercase().as_str(),
+ "yes" | "no" | "on" | "off" | "y" | "n"
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde::Deserialize;
+ #[derive(Debug, Deserialize)]
+ struct Document {
+ #[serde(default)]
+ entries: std::collections::BTreeMap,
+ }
+ #[derive(Debug, Default, Deserialize)]
+ struct Entry {
+ #[serde(default)]
+ config: std::collections::BTreeMap,
+ }
+
+ #[test]
+ fn null_objects_are_rejected_but_user_data_null_is_retained() {
+ for source in [
+ "entries: null",
+ "entries: {x: null}",
+ "entries: {x: {config: null}}",
+ ] {
+ assert!(
+ from_str_with_object_paths::(
+ source,
+ &["entries", "entries.*", "entries.*.config"]
+ )
+ .is_err(),
+ "{source}"
+ );
+ }
+ let parsed: Document = from_str_with_object_paths(
+ "entries: {x: {config: {optional: null}}}",
+ &["entries", "entries.*", "entries.*.config"],
+ )
+ .unwrap();
+ assert!(parsed.entries["x"].config["optional"].is_null());
+ }
+ #[test]
+ fn duplicate_fields_are_rejected() {
+ assert!(from_str::("entries: {}\nentries: {x: {}}").is_err());
+ }
+ #[test]
+ fn empty_documents_do_not_become_default_objects() {
+ for source in ["", "# comment only", "null", "~"] {
+ assert!(from_str::(source).is_err(), "{source}");
+ }
+ assert!(from_str::("{}").is_ok());
+ }
+
+ #[test]
+ fn object_paths_preserve_optional_nulls_and_visit_sequence_elements() {
+ #[derive(Debug, Deserialize)]
+ struct Profile {
+ optional: Option,
+ endpoints: Vec,
+ }
+ let paths = ["endpoints.*", "endpoints.*.config", "optional.config"];
+ let parsed: Profile = from_str_with_object_paths(
+ "optional: null\nendpoints: [{config: {data: null}}]",
+ &paths,
+ )
+ .unwrap();
+ assert!(parsed.optional.is_none());
+ assert!(parsed.endpoints[0].config["data"].is_null());
+ for source in [
+ "optional: null\nendpoints: [null]",
+ "optional: {config: null}\nendpoints: []",
+ ] {
+ assert!(
+ from_str_with_object_paths::(source, &paths).is_err(),
+ "{source}"
+ );
+ }
+ let source = "entries: {'key.with.dots': null}";
+ let error = from_str_with_object_paths::(source, &["entries.*"]).unwrap_err();
+ assert!(error.to_string().contains("key.with.dots"));
+ }
+ #[test]
+ fn legacy_reader_sensitive_strings_are_quoted_without_changing_values() {
+ let value = serde_json::json!({"versions": ["2025-11-25"], "timestamp": "2026-01-01T00:00:00Z", "number": "1_000", "boolean": "yes", "duration": "1.500s", "actual_number": 1000, "data": null});
+ let yaml = to_string(&value).unwrap();
+ for string in ["2025-11-25", "2026-01-01T00:00:00Z", "1_000", "yes"] {
+ assert!(
+ yaml.contains(&format!("\"{string}\"")) || yaml.contains(&format!("'{string}'")),
+ "{yaml}"
+ );
+ }
+ assert_eq!(from_str::(&yaml).unwrap(), value);
+ }
+}
diff --git a/crates/openshell-prover/src/accepted_risks.rs b/crates/openshell-prover/src/accepted_risks.rs
index 8c28a44188..52c77947f5 100644
--- a/crates/openshell-prover/src/accepted_risks.rs
+++ b/crates/openshell-prover/src/accepted_risks.rs
@@ -57,9 +57,10 @@ pub fn load_accepted_risks(path: &Path) -> Result> {
let contents = std::fs::read_to_string(path)
.into_diagnostic()
.wrap_err_with(|| format!("reading accepted risks {}", path.display()))?;
- let raw: AcceptedRisksFile = serde_yml::from_str(&contents)
- .into_diagnostic()
- .wrap_err("parsing accepted risks YAML")?;
+ let raw: AcceptedRisksFile =
+ openshell_policy_schema::yaml::from_str_with_object_paths(&contents, &["accepted_risks.*"])
+ .into_diagnostic()
+ .wrap_err("parsing accepted risks YAML")?;
Ok(raw
.accepted_risks
diff --git a/crates/openshell-prover/src/containment.rs b/crates/openshell-prover/src/containment.rs
index 9a9c9cb72a..d2bd213e95 100644
--- a/crates/openshell-prover/src/containment.rs
+++ b/crates/openshell-prover/src/containment.rs
@@ -966,12 +966,35 @@ fn endpoint_matches_connection(endpoint: &Endpoint, action: &SymbolicAction) ->
}
fn endpoint_path_matches(endpoint: &Endpoint, action: &SymbolicAction) -> Bool {
- let path = if endpoint.path.is_empty() {
- "**"
+ let pattern = endpoint.path.as_str();
+ // Mirror openshell_core::endpoint_path, not the REST rule glob matcher.
+ if matches!(pattern, "" | "**" | "/**") {
+ return Bool::from_bool(true);
+ }
+ if let Some(prefix) = pattern.strip_suffix("/**") {
+ // The entire prefix is literal, including any wildcard characters.
+ return Bool::or(&[
+ action.path.eq(prefix),
+ Z3String::from_str(&format!("{prefix}/"))
+ .unwrap()
+ .prefix(&action.path),
+ ]);
+ }
+ if glob::Pattern::new(pattern).is_err() {
+ return action.path.eq(pattern);
+ }
+ // Rust glob consumes the slash after ** while parsing. A terminal **/
+ // therefore matches the rest of the path, even without a trailing slash.
+ let pattern = if pattern.ends_with("/**/") {
+ &pattern[..pattern.len() - 1]
} else {
- &endpoint.path
+ pattern
};
- action.path.regex_matches(&glob_regex(path, "/"))
+ // Validation excludes ?, brackets, braces and escapes. For the remaining
+ // Rust glob language, a single star also crosses path separators.
+ action
+ .path
+ .regex_matches(&path_glob_regex(pattern, &Regexp::full()))
}
fn method_and_path_match(method: &str, path: &str, action: &SymbolicAction) -> Bool {
@@ -2136,7 +2159,7 @@ fn glob_regex(pattern: &str, separator: &str) -> Regexp {
return Regexp::full();
}
if separator == "/" {
- return path_glob_regex(pattern);
+ return path_glob_regex(pattern, &non_separator_regex("/").star());
}
let mut parts = Vec::new();
let mut chars = pattern.chars().peekable();
@@ -2159,7 +2182,7 @@ fn glob_regex(pattern: &str, separator: &str) -> Regexp {
}
}
-fn path_glob_regex(pattern: &str) -> Regexp {
+fn path_glob_regex(pattern: &str, single_star: &Regexp) -> Regexp {
let mut parts = Vec::new();
let mut segments = pattern.split('/').peekable();
while let Some(segment) = segments.next() {
@@ -2180,9 +2203,10 @@ fn path_glob_regex(pattern: &str) -> Regexp {
});
} else {
for character in segment.chars() {
- // Embedded stars, including **, cannot cross a separator.
+ // REST rule stars stay within a segment; endpoint selector
+ // stars use the unrestricted Rust glob match language.
parts.push(if character == '*' {
- non_separator_regex("/").star()
+ single_star.clone()
} else {
Regexp::literal(&character.to_string())
});
@@ -3316,6 +3340,71 @@ network_policies:
}
}
+ #[test]
+ fn endpoint_paths_match_the_canonical_runtime_matcher() {
+ for pattern in [
+ "",
+ "**",
+ "/**",
+ "/a",
+ "/a/*",
+ "/a/*/b",
+ "/a/**",
+ "/a/**/b",
+ "/a/**/**/b",
+ "/a/**/",
+ "/**/",
+ "/a/**/**/",
+ "/a/**/b/**/",
+ "/a/*/**/",
+ "/a*/**",
+ "/a/**/**",
+ "/a/x**/b",
+ "/a/**x/b",
+ "/a/***/b",
+ ] {
+ let endpoint: Endpoint = serde_json::from_value(serde_json::json!({
+ "host": "api.example.com", "port": 443, "path": pattern,
+ }))
+ .unwrap();
+ let mut action = symbolic_action("endpoint_path_parity");
+ for path in [
+ pattern,
+ "/",
+ "/a",
+ "/a/",
+ "/a/b",
+ "/a/x/b",
+ "/a/x/y/b",
+ "/a/x/y/",
+ "/a/xb/b",
+ "/a/xx/b",
+ "/ab/c",
+ "/a*",
+ "/a*/b",
+ "/a/**",
+ "/a/**/b",
+ "/a/é/b",
+ "/a/汉/b",
+ "/a/e\u{301}/b",
+ "/a/😀/b",
+ ] {
+ action.path = Z3String::from_str(path).unwrap();
+ let solver = Solver::new();
+ solver.assert(endpoint_path_matches(&endpoint, &action));
+ assert_eq!(
+ solver.check(),
+ if openshell_core::endpoint_path::matches(pattern, path) {
+ SatResult::Sat
+ } else {
+ SatResult::Unsat
+ },
+ "pattern={pattern:?} path={path:?}"
+ );
+ }
+ }
+ }
+
#[test]
fn path_globs_match_the_runtime_builtin() {
let mut runtime = regorus::Engine::new();
diff --git a/crates/openshell-prover/src/credentials.rs b/crates/openshell-prover/src/credentials.rs
index 6c9a1a74f3..a7be2d559f 100644
--- a/crates/openshell-prover/src/credentials.rs
+++ b/crates/openshell-prover/src/credentials.rs
@@ -268,9 +268,10 @@ pub fn load_credentials(path: &Path) -> Result> {
let contents = std::fs::read_to_string(path)
.into_diagnostic()
.wrap_err_with(|| format!("reading credentials file {}", path.display()))?;
- let raw: CredentialsFile = serde_yml::from_str(&contents)
- .into_diagnostic()
- .wrap_err("parsing credentials YAML")?;
+ let raw: CredentialsFile =
+ openshell_policy_schema::yaml::from_str_with_object_paths(&contents, &["credentials.*"])
+ .into_diagnostic()
+ .wrap_err("parsing credentials YAML")?;
Ok(raw
.credentials
@@ -286,9 +287,16 @@ pub fn load_credentials(path: &Path) -> Result> {
}
fn parse_api_registry(contents: &str, source: &str) -> Result {
- let raw: ApiRegistryDef = serde_yml::from_str(contents)
- .into_diagnostic()
- .wrap_err_with(|| format!("parsing API registry {source}"))?;
+ let raw: ApiRegistryDef = openshell_policy_schema::yaml::from_str_with_object_paths(
+ contents,
+ &[
+ "scope_capabilities",
+ "scope_capabilities.*.*",
+ "action_risk",
+ ],
+ )
+ .into_diagnostic()
+ .wrap_err_with(|| format!("parsing API registry {source}"))?;
let scope_capabilities = raw
.scope_capabilities
diff --git a/crates/openshell-prover/src/registry.rs b/crates/openshell-prover/src/registry.rs
index 63a3fce3bc..c425b2e9be 100644
--- a/crates/openshell-prover/src/registry.rs
+++ b/crates/openshell-prover/src/registry.rs
@@ -194,9 +194,12 @@ impl BinaryRegistry {
// ---------------------------------------------------------------------------
fn parse_binary_capability(contents: &str, source: &str) -> Result {
- let raw: BinaryCapabilityDef = serde_yml::from_str(contents)
- .into_diagnostic()
- .wrap_err_with(|| format!("parsing binary descriptor {source}"))?;
+ let raw: BinaryCapabilityDef = openshell_policy_schema::yaml::from_str_with_object_paths(
+ contents,
+ &["protocols.*", "protocols.*.actions.*"],
+ )
+ .into_diagnostic()
+ .wrap_err_with(|| format!("parsing binary descriptor {source}"))?;
let protocols = raw
.protocols
diff --git a/crates/openshell-prover/tests/runtime_parity.rs b/crates/openshell-prover/tests/runtime_parity.rs
index 198198bdae..81b4557cb4 100644
--- a/crates/openshell-prover/tests/runtime_parity.rs
+++ b/crates/openshell-prover/tests/runtime_parity.rs
@@ -36,6 +36,19 @@ fn runtime_engine_with_identity(policy: &str, require_binary_identity: bool) ->
);
let mut engine = Engine::new();
+ // The runtime registers this native builtin for endpoint selectors only.
+ engine
+ .add_extension(
+ "openshell.endpoint_path_matches".into(),
+ 2,
+ Box::new(|params: Vec| {
+ Ok(Value::from(openshell_core::endpoint_path::matches(
+ params[0].as_string()?,
+ params[1].as_string()?,
+ )))
+ }),
+ )
+ .expect("runtime endpoint path matcher should register");
engine
.add_policy("sandbox-policy.rego".into(), SANDBOX_POLICY_REGO.into())
.expect("runtime Rego should compile");
@@ -46,6 +59,16 @@ fn runtime_engine_with_identity(policy: &str, require_binary_identity: bool) ->
}
fn runtime_input(binary: &str, ancestors: &[&str], host: &str, method: &str) -> Value {
+ runtime_input_with_path(binary, ancestors, host, method, "/")
+}
+
+fn runtime_input_with_path(
+ binary: &str,
+ ancestors: &[&str],
+ host: &str,
+ method: &str,
+ path: &str,
+) -> Value {
serde_json::from_value(json!({
"exec": {
"path": binary,
@@ -58,7 +81,7 @@ fn runtime_input(binary: &str, ancestors: &[&str], host: &str, method: &str) ->
},
"request": {
"method": method,
- "path": "/",
+ "path": path,
"query_params": {},
},
}))
@@ -85,6 +108,133 @@ fn eval_array_len(engine: &mut Engine, input: &Value, rule: &str) -> usize {
}
}
+fn endpoint_path_policy(selector: &str, rule_path: &str) -> String {
+ json!({
+ "version": 1,
+ "network_policies": {"egress": {
+ "binaries": [{"path": "/usr/bin/curl"}],
+ "endpoints": [{
+ "host": "api.example.com", "ports": [443],
+ "protocol": "rest", "enforcement": "enforce", "path": selector,
+ "rules": [{"allow": {"method": "GET", "path": rule_path}}]
+ }]
+ }}
+ })
+ .to_string()
+}
+
+fn assert_runtime_path_expansion_replays(boundary: &str, candidate: &str, path: &str) {
+ let input = runtime_input_with_path("/usr/bin/curl", &[], "api.example.com", "GET", path);
+ assert!(eval_bool(
+ &mut runtime_engine(candidate),
+ &input,
+ "data.openshell.sandbox.allow_request"
+ ));
+ assert!(!eval_bool(
+ &mut runtime_engine(boundary),
+ &input,
+ "data.openshell.sandbox.allow_request"
+ ));
+
+ let result = check(boundary, candidate);
+ let CheckResult::Exceeds(evidence) = result else {
+ panic!("runtime expansion must exceed the boundary: {result:?}");
+ };
+ let Counterexample::Network {
+ binary,
+ ancestor_binary,
+ binary_identity_required,
+ host,
+ port,
+ method: Some(method),
+ path: Some(path),
+ ..
+ } = evidence.counterexample()
+ else {
+ panic!("expected a REST counterexample");
+ };
+ let input = serde_json::from_value(json!({
+ "exec": {
+ "path": binary.as_deref().unwrap_or(""),
+ "ancestors": ancestor_binary.iter().collect::>(),
+ "cmdline_paths": [],
+ },
+ "network": {"host": host, "port": port},
+ "request": {"method": method, "path": path, "query_params": {}}
+ }))
+ .unwrap();
+ assert!(eval_bool(
+ &mut runtime_engine_with_identity(candidate, *binary_identity_required),
+ &input,
+ "data.openshell.sandbox.allow_request"
+ ));
+ assert!(!eval_bool(
+ &mut runtime_engine_with_identity(boundary, *binary_identity_required),
+ &input,
+ "data.openshell.sandbox.allow_request"
+ ));
+}
+
+#[test]
+fn endpoint_selector_single_star_crosses_path_separators() {
+ assert_runtime_path_expansion_replays(
+ &endpoint_path_policy("", "/v1/*"),
+ &endpoint_path_policy("/v1/*", "/**"),
+ "/v1/a/b",
+ );
+}
+
+#[test]
+fn endpoint_selector_subtree_includes_the_prefix_itself() {
+ assert_runtime_path_expansion_replays(
+ &endpoint_path_policy("", "/p/**"),
+ &endpoint_path_policy("/p/**", "/**"),
+ "/p",
+ );
+}
+
+#[test]
+fn endpoint_selector_scoped_denies_match_runtime() {
+ for (selector, path) in [("/v1/*", "/v1/a/b"), ("/p/**", "/p")] {
+ let mut boundary: serde_json::Value =
+ serde_json::from_str(&endpoint_path_policy("", "/**")).unwrap();
+ let endpoints = boundary["network_policies"]["egress"]["endpoints"]
+ .as_array_mut()
+ .unwrap();
+ let mut deny = endpoints[0].clone();
+ deny["path"] = json!(selector);
+ deny["deny_rules"] = json!([{"method": "GET", "path": "/**"}]);
+ endpoints.push(deny);
+ assert_runtime_path_expansion_replays(
+ &boundary.to_string(),
+ &endpoint_path_policy("", path),
+ path,
+ );
+ }
+}
+
+#[test]
+fn endpoint_selector_contained_policies_remain_within() {
+ for (boundary_selector, boundary_rule, candidate_selector, candidate_rule, path) in [
+ ("/v1/*", "/**", "", "/v1/**", "/v1/a/b"),
+ ("/p/**", "/**", "", "/p", "/p"),
+ ("/p/**", "/**", "/p/**", "/**", "/p/a"),
+ ] {
+ let boundary = endpoint_path_policy(boundary_selector, boundary_rule);
+ let candidate = endpoint_path_policy(candidate_selector, candidate_rule);
+ let input = runtime_input_with_path("/usr/bin/curl", &[], "api.example.com", "GET", path);
+ for policy in [&boundary, &candidate] {
+ assert!(eval_bool(
+ &mut runtime_engine(policy),
+ &input,
+ "data.openshell.sandbox.allow_request"
+ ));
+ }
+ let result = check(&boundary, &candidate);
+ assert!(matches!(result, CheckResult::Within(_)), "{result:?}");
+ }
+}
+
#[test]
fn underscore_host_counterexample_replays_at_runtime() {
let boundary = "version: 1\n";
diff --git a/crates/openshell-providers/README.md b/crates/openshell-providers/README.md
index f0b5b79233..2a9b686019 100644
--- a/crates/openshell-providers/README.md
+++ b/crates/openshell-providers/README.md
@@ -13,6 +13,9 @@ out of the CLI and gateway control flow.
- Discover local credentials from environment variables and known config files.
- Normalize discovered data into provider records.
- Keep provider-specific parsing rules in provider modules.
+- Use the shared authored-YAML compatibility boundary for profile imports and
+ exports: reject duplicate keys and null objects without coercion, retain
+ optional nulls, and preserve string types for YAML 1.1 readers.
- Avoid logging credential values.
## Non-Responsibilities
diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs
index c93206c733..1f7655aae3 100644
--- a/crates/openshell-providers/src/profiles.rs
+++ b/crates/openshell-providers/src/profiles.rs
@@ -1974,7 +1974,31 @@ fn graphql_operation_from_proto(operation: &GraphqlOperation) -> GraphqlOperatio
}
pub fn parse_profile_yaml(input: &str) -> Result {
- Ok(serde_yml::from_str::(input)?)
+ Ok(openshell_core::yaml::from_str_with_object_paths::<
+ ProviderTypeProfile,
+ >(
+ input,
+ &[
+ "annotations",
+ "discovery",
+ "credentials.*",
+ "files.*",
+ "endpoints.*",
+ "binaries.*",
+ "credentials.*.refresh.material.*",
+ "credentials.*.refresh.additional_outputs.*",
+ "credentials.*.token_grant.audience_overrides.*",
+ "endpoints.*.mcp",
+ "endpoints.*.graphql_persisted_queries",
+ "endpoints.*.graphql_persisted_queries.*",
+ "endpoints.*.rules.*",
+ "endpoints.*.rules.*.allow.query",
+ "endpoints.*.rules.*.allow.params",
+ "endpoints.*.deny_rules.*",
+ "endpoints.*.deny_rules.*.query",
+ "endpoints.*.deny_rules.*.params",
+ ],
+ )?)
}
pub fn parse_profile_json(input: &str) -> Result {
@@ -1982,7 +2006,7 @@ pub fn parse_profile_json(input: &str) -> Result Result {
- Ok(serde_yml::to_string(profile)?)
+ Ok(openshell_core::yaml::to_string(profile)?)
}
pub fn profile_to_json(profile: &ProviderTypeProfile) -> Result {
@@ -1990,7 +2014,7 @@ pub fn profile_to_json(profile: &ProviderTypeProfile) -> Result Result {
- Ok(serde_yml::to_string(profiles)?)
+ Ok(openshell_core::yaml::to_string(profiles)?)
}
pub fn profiles_to_json(profiles: &[ProviderTypeProfile]) -> Result {
@@ -3795,9 +3819,18 @@ credentials:
let exact = ProviderTypeProfile::from_proto(&proto);
let yaml = profile_to_yaml(&exact).unwrap();
- assert!(yaml.contains("refresh_before: \"0s\""), "{yaml}");
- assert!(yaml.contains("max_lifetime: \"1.500s\""), "{yaml}");
- assert!(yaml.contains("cache_ttl: \"0.500s\""), "{yaml}");
+ // Check string values rather than the emitter's choice of quote style.
+ let document: serde_yml::Value = serde_yml::from_str(&yaml).unwrap();
+ let credential = &document["credentials"][0];
+ assert_eq!(credential["refresh"]["refresh_before"].as_str(), Some("0s"));
+ assert_eq!(
+ credential["refresh"]["max_lifetime"].as_str(),
+ Some("1.500s")
+ );
+ assert_eq!(
+ credential["token_grant"]["cache_ttl"].as_str(),
+ Some("0.500s")
+ );
assert!(!yaml.contains("refresh_before_seconds"));
assert!(!yaml.contains("max_lifetime_seconds"));
assert!(!yaml.contains("cache_ttl_seconds"));
@@ -3813,6 +3846,24 @@ credentials:
);
}
+ #[test]
+ fn yaml_profile_rejects_null_objects_and_duplicate_fields() {
+ for field in [
+ "annotations: null",
+ "discovery: null",
+ "endpoints: [null]",
+ "endpoints: [{host: example.com, port: 443, rules: [{allow: {query: null}}]}]",
+ "endpoints: [{host: example.com, port: 443, rules: [{allow: {params: null}}]}]",
+ "endpoints: [{host: example.com, port: 443, graphql_persisted_queries: null}]",
+ ] {
+ let yaml = format!("id: sample\ndisplay_name: Sample\n{field}\n");
+ assert!(parse_profile_yaml(&yaml).is_err(), "{yaml}");
+ }
+ assert!(parse_profile_yaml("id: first\nid: second\ndisplay_name: Sample\n").is_err());
+ // Existing optional fields still accept null as absence.
+ assert!(parse_profile_yaml("id: sample\ndisplay_name: Sample\nendpoints: [{host: example.com, port: 443, rules: [{allow: null}]}]\n").is_ok());
+ }
+
#[test]
fn canonical_zero_max_lifetime_is_rejected() {
let profile = parse_profile_yaml(
diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs
index fbd2df886e..f6e08e78aa 100644
--- a/crates/openshell-sandbox/src/boundary_server.rs
+++ b/crates/openshell-sandbox/src/boundary_server.rs
@@ -82,6 +82,11 @@ mod linux {
const MAX_CONTROL_CONNECTIONS: usize = 128;
const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096;
const MAX_RETAINED_EXEC_PROCESSES: usize = 64;
+ // Three stdio pipes require six descriptors before fork, and Rust's
+ // process launcher may briefly need additional bookkeeping descriptors.
+ // Reclaim closed mediated sockets before each serialized spawn so stale
+ // socket sources cannot consume that bounded launch budget.
+ const EXEC_SPAWN_DESCRIPTOR_HEADROOM: usize = 16;
// NVML may traverse the persistenced socket directory during initialization;
// WSL2 supplies GPU libraries under /usr/lib/wsl and the /dev/dxg device.
@@ -1398,6 +1403,29 @@ mod linux {
status: Arc>>,
}
+ async fn retain_exec_completion(
+ process: Arc,
+ session: Arc,
+ status: Arc>>,
+ ) {
+ let exit_status = match process.wait().await {
+ Ok(exit_status) => ExitStatusWire::from(exit_status),
+ Err(error) => {
+ // A process wait error is terminal per the BoundaryProcess
+ // contract. Retaining `None` here makes this handle look live
+ // forever and can permanently exhaust exec admission.
+ tracing::warn!(%error, "Exec process wait failed; recording terminal status");
+ ExitStatusWire::Exited(74)
+ }
+ };
+ *lock(&status) = Some(exit_status);
+ let exit_code = match exit_status {
+ ExitStatusWire::Exited(code) => code,
+ ExitStatusWire::Signaled(signal) => 128 + signal,
+ };
+ let _ = session.finish_remote(exit_code, false).await;
+ }
+
#[derive(Default)]
struct ExecRequestLedger {
requests: std::collections::HashSet,
@@ -2162,6 +2190,14 @@ mod linux {
}
requests.reserve(request_id, expires_at)?;
drop(requests);
+ self.network_broker
+ .ensure_descriptor_headroom(EXEC_SPAWN_DESCRIPTOR_HEADROOM)
+ .map_err(|error| {
+ guest_error(
+ BoundaryErrorKind::Unavailable,
+ format!("exec descriptor headroom unavailable: {error}"),
+ )
+ })?;
let session = self
.process_runtime
.block_on(executor.exec(spec.into()))
@@ -2202,18 +2238,7 @@ mod linux {
let wait_session = retained.clone();
let wait_status = status.clone();
self.process_runtime.spawn(async move {
- if let Ok(exit_status) = wait_process.wait().await {
- *lock(&wait_status) = Some(ExitStatusWire::from(exit_status));
- let exit_code = match exit_status {
- openshell_isolation_interface::contract::BoundaryExitStatus::Exited(
- code,
- ) => code,
- openshell_isolation_interface::contract::BoundaryExitStatus::Signaled(
- signal,
- ) => 128 + signal,
- };
- let _ = wait_session.finish_remote(exit_code, false).await;
- }
+ retain_exec_completion(wait_process, wait_session, wait_status).await;
});
let handle = ExecHandle {
request_id: request_id.to_string(),
@@ -3820,6 +3845,52 @@ mod linux {
};
use rcgen::{KeyPair, PKCS_ED25519};
+ struct FailedWaitProcess;
+
+ #[async_trait::async_trait]
+ impl BoundaryProcess for FailedWaitProcess {
+ async fn wait(
+ &self,
+ ) -> Result<
+ openshell_isolation_interface::contract::BoundaryExitStatus,
+ openshell_isolation_interface::contract::BackendError,
+ > {
+ Err(
+ openshell_isolation_interface::contract::BackendError::Terminated(
+ "test boundary loss".to_string(),
+ ),
+ )
+ }
+
+ async fn signal(
+ &self,
+ _signal: openshell_isolation_interface::contract::BoundarySignal,
+ ) -> Result<(), openshell_isolation_interface::contract::BackendError> {
+ Ok(())
+ }
+
+ async fn terminate(
+ &self,
+ ) -> Result<(), openshell_isolation_interface::contract::BackendError> {
+ Ok(())
+ }
+ }
+
+ #[tokio::test]
+ async fn exec_wait_failure_records_reclaimable_terminal_status() {
+ let session = MainSession::inert();
+ let mut output = session.subscribe();
+ let status = Arc::new(Mutex::new(None));
+
+ retain_exec_completion(Arc::new(FailedWaitProcess), session, status.clone()).await;
+
+ assert_eq!(*lock(&status), Some(ExitStatusWire::Exited(74)));
+ assert!(matches!(
+ output.recv().await.expect("terminal exec output"),
+ MainOutput::Exit(74)
+ ));
+ }
+
#[test]
fn exec_tombstones_expire_without_a_lifetime_limit() {
let mut ledger = ExecRequestLedger::default();
diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs
index 8988ae3328..3266b55726 100644
--- a/crates/openshell-sandbox/src/network_broker.rs
+++ b/crates/openshell-sandbox/src/network_broker.rs
@@ -174,7 +174,7 @@ struct NotificationQueues {
pending: mpsc::Sender,
dns_relay: DnsRelay,
active_opens: Arc,
- retained_socket_capacity: usize,
+ descriptor_soft_limit: usize,
decision_timeout: Duration,
}
@@ -185,6 +185,8 @@ pub struct NetworkBroker {
workload_frozen: Arc,
pending: Arc>>,
pending_dns: Arc>>,
+ registry: Arc>,
+ descriptor_soft_limit: usize,
dns_address: SocketAddr,
healthy: Arc,
}
@@ -231,7 +233,7 @@ impl NetworkBroker {
let active_opens = Arc::new(AtomicUsize::new(0));
let dns_relay = start_dns_relay(dns_address, pending_dns_tx)?;
let dns_address = dns_relay.address;
- let retained_socket_capacity = retained_socket_capacity()?;
+ let descriptor_soft_limit = descriptor_soft_limit()?;
let registry = Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?));
let provider_files = crate::provider_files::ProviderFiles::default();
let workload_frozen = Arc::new(AtomicBool::new(false));
@@ -243,11 +245,12 @@ impl NetworkBroker {
pending: pending_tx,
dns_relay,
active_opens,
- retained_socket_capacity,
+ descriptor_soft_limit,
decision_timeout,
};
let healthy = Arc::new(AtomicBool::new(true));
let broker_healthy = healthy.clone();
+ let broker_registry = Arc::clone(®istry);
std::thread::Builder::new()
.name("openshell-network-broker".to_string())
.spawn(move || {
@@ -272,7 +275,7 @@ impl NetworkBroker {
// broker keeps mediating the rest.
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
dispatch_notification(
- Arc::clone(®istry),
+ Arc::clone(&broker_registry),
Arc::clone(&listener),
notification,
queues.clone(),
@@ -314,11 +317,21 @@ impl NetworkBroker {
workload_frozen,
pending: Arc::new(tokio::sync::Mutex::new(pending_rx)),
pending_dns: Arc::new(tokio::sync::Mutex::new(pending_dns_rx)),
+ registry,
+ descriptor_soft_limit,
dns_address,
healthy,
})
}
+ /// Reclaim closed workload sockets before an operation that needs several
+ /// descriptors at once. Socket creation already protects its own
+ /// descriptor, but exec startup creates multiple pipes without first
+ /// issuing another mediated socket syscall.
+ pub(crate) fn ensure_descriptor_headroom(&self, required: usize) -> io::Result<()> {
+ ensure_descriptor_headroom(&self.registry, self.descriptor_soft_limit, required)
+ }
+
/// Record whether the boundary has stopped the workload for supervisor
/// recovery. While frozen, workload requests to send `SIGCONT` are refused
/// so a process that was not yet stopped cannot resume the others. Set
@@ -571,7 +584,7 @@ fn dispatch_notification(
®istry,
&listener,
notification,
- queues.retained_socket_capacity,
+ queues.descriptor_soft_limit,
);
}
if syscall == libc::SYS_connect {
@@ -632,7 +645,7 @@ fn create_socket(
registry: &Mutex,
listener: &NotificationListener,
notification: Notification,
- retained_socket_capacity: usize,
+ descriptor_soft_limit: usize,
) -> io::Result<()> {
let domain = i32::try_from(notification.args[0])
.map_err(|_| io::Error::from_raw_os_error(libc::EAFNOSUPPORT))?;
@@ -663,7 +676,7 @@ fn create_socket(
// Reclaim stale descriptors before the broker exhausts its process limit.
// Connected sockets remain in the metadata registry without consuming
// this broker-owned descriptor budget.
- if let Err(error) = prepare_registry_for_socket(registry, retained_socket_capacity) {
+ if let Err(error) = prepare_registry_for_socket(registry, descriptor_soft_limit) {
return listener.respond_errno(notification.id, error_to_errno(&error));
}
// SAFETY: arguments were reduced to the supported native INET matrix. A
@@ -705,7 +718,7 @@ fn create_socket(
Ok(())
}
-fn retained_socket_capacity() -> io::Result {
+fn descriptor_soft_limit() -> io::Result {
let mut limit = libc::rlimit {
rlim_cur: 0,
rlim_max: 0,
@@ -714,30 +727,75 @@ fn retained_socket_capacity() -> io::Result {
if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut limit) } < 0 {
return Err(io::Error::last_os_error());
}
- let soft_limit = usize::try_from(limit.rlim_cur).unwrap_or(usize::MAX);
- let open_descriptors = std::fs::read_dir("/proc/self/fd")?.count();
- Ok(retained_socket_capacity_for_limit(
- soft_limit,
- open_descriptors,
- ))
+ Ok(usize::try_from(limit.rlim_cur).unwrap_or(usize::MAX))
}
-fn retained_socket_capacity_for_limit(soft_limit: usize, open_descriptors: usize) -> usize {
- soft_limit
- .saturating_sub(open_descriptors)
- .saturating_sub(SOCKET_FD_HEADROOM)
- .clamp(1, SOCKET_CAPACITY)
+fn open_descriptor_count() -> io::Result {
+ std::fs::read_dir("/proc/self/fd").map(Iterator::count)
+}
+
+fn descriptor_headroom_exhausted(
+ soft_limit: usize,
+ open_descriptors: usize,
+ required: usize,
+) -> bool {
+ open_descriptors.saturating_add(required) > soft_limit
}
fn prepare_registry_for_socket(
registry: &Mutex,
- retained_socket_capacity: usize,
+ descriptor_soft_limit: usize,
+) -> io::Result<()> {
+ // Descriptor use changes after broker startup as control streams and execs
+ // come and go. Recompute it for each socket request so retained pre-connect
+ // sockets cannot consume the headroom reserved for those control paths.
+ prepare_registry_for_socket_with_count(registry, descriptor_soft_limit, open_descriptor_count)
+}
+
+fn prepare_registry_for_socket_with_count(
+ registry: &Mutex,
+ descriptor_soft_limit: usize,
+ mut open_descriptors: impl FnMut() -> io::Result,
+) -> io::Result<()> {
+ let required = SOCKET_FD_HEADROOM.saturating_add(1);
+ let mut registry = lock(registry);
+ if registry.is_full()
+ || descriptor_headroom_exhausted(descriptor_soft_limit, open_descriptors()?, required)
+ {
+ collect_closed_socket_entries_locked(&mut registry)?;
+ }
+ if registry.is_full()
+ || descriptor_headroom_exhausted(descriptor_soft_limit, open_descriptors()?, required)
+ {
+ return Err(io::Error::from_raw_os_error(libc::EMFILE));
+ }
+ Ok(())
+}
+
+fn ensure_descriptor_headroom(
+ registry: &Mutex,
+ descriptor_soft_limit: usize,
+ required: usize,
+) -> io::Result<()> {
+ ensure_descriptor_headroom_with_count(
+ registry,
+ descriptor_soft_limit,
+ required,
+ open_descriptor_count,
+ )
+}
+
+fn ensure_descriptor_headroom_with_count(
+ registry: &Mutex,
+ descriptor_soft_limit: usize,
+ required: usize,
+ mut open_descriptors: impl FnMut() -> io::Result,
) -> io::Result<()> {
let mut registry = lock(registry);
- if registry.is_full() || registry.retained_preconnect_count() >= retained_socket_capacity {
+ if descriptor_headroom_exhausted(descriptor_soft_limit, open_descriptors()?, required) {
collect_closed_socket_entries_locked(&mut registry)?;
}
- if registry.is_full() || registry.retained_preconnect_count() >= retained_socket_capacity {
+ if descriptor_headroom_exhausted(descriptor_soft_limit, open_descriptors()?, required) {
return Err(io::Error::from_raw_os_error(libc::EMFILE));
}
Ok(())
@@ -1761,18 +1819,15 @@ mod tests {
use std::os::unix::net::{UnixListener, UnixStream};
#[test]
- fn retained_socket_capacity_reserves_process_descriptor_headroom() {
- assert_eq!(retained_socket_capacity_for_limit(1_024, 24), 936);
- assert_eq!(retained_socket_capacity_for_limit(128, 32), 32);
- assert_eq!(retained_socket_capacity_for_limit(64, 0), 1);
- assert_eq!(
- retained_socket_capacity_for_limit(usize::MAX, 0),
- SOCKET_CAPACITY
- );
+ fn descriptor_budget_reserves_process_headroom_from_current_usage() {
+ assert!(!descriptor_headroom_exhausted(1_024, 959, 65));
+ assert!(descriptor_headroom_exhausted(1_024, 960, 65));
+ assert!(descriptor_headroom_exhausted(64, 0, 65));
+ assert!(!descriptor_headroom_exhausted(usize::MAX, 4_096, 65));
}
#[test]
- fn retained_socket_limit_reclaims_stale_entry_before_opening_another_socket() {
+ fn descriptor_pressure_reclaims_stale_socket_after_ambient_usage_grows() {
// SAFETY: socket returns one newly owned descriptor on success.
let fd = unsafe {
libc::socket(
@@ -1798,13 +1853,19 @@ mod tests {
assert_eq!(registry.retained_preconnect_count(), 1);
let registry = Mutex::new(registry);
- prepare_registry_for_socket(®istry, 1).unwrap();
+ let mut observed = std::collections::VecDeque::from([64, 63]);
+ prepare_registry_for_socket_with_count(®istry, 128, || {
+ observed
+ .pop_front()
+ .ok_or_else(|| io::Error::other("unexpected descriptor recount"))
+ })
+ .unwrap();
assert!(lock(®istry).is_empty());
}
#[test]
- fn retained_socket_limit_does_not_cap_connected_metadata() {
+ fn exec_headroom_reclaims_stale_socket_before_pipe_allocation() {
// SAFETY: socket returns one newly owned descriptor on success.
let fd = unsafe {
libc::socket(
@@ -1825,6 +1886,42 @@ mod tests {
};
let mut registry = SocketRegistry::new(1, 2).unwrap();
let tentative = registry.stage(socket, metadata).unwrap();
+ registry.commit(tentative).unwrap();
+
+ let registry = Mutex::new(registry);
+ let mut observed = std::collections::VecDeque::from([120, 112]);
+ ensure_descriptor_headroom_with_count(®istry, 128, 16, || {
+ observed
+ .pop_front()
+ .ok_or_else(|| io::Error::other("unexpected descriptor recount"))
+ })
+ .unwrap();
+
+ assert!(lock(®istry).is_empty());
+ }
+
+ #[test]
+ fn descriptor_budget_does_not_cap_connected_metadata() {
+ // SAFETY: socket returns one newly owned descriptor on success.
+ let fd = unsafe {
+ libc::socket(
+ libc::AF_INET,
+ libc::SOCK_STREAM | libc::SOCK_CLOEXEC,
+ libc::IPPROTO_TCP,
+ )
+ };
+ assert!(fd >= 0, "socket: {}", io::Error::last_os_error());
+ // SAFETY: successful socket returned one owned descriptor.
+ let socket = unsafe { OwnedFd::from_raw_fd(fd) };
+ let metadata = SocketMetadata {
+ family: InetFamily::V4,
+ kind: InetKind::Tcp,
+ close_on_exec: true,
+ nonblocking: false,
+ creator_generation: 1,
+ };
+ let mut registry = SocketRegistry::new(1, 1).unwrap();
+ let tentative = registry.stage(socket, metadata).unwrap();
registry
.commit_with_state(
tentative,
@@ -1834,10 +1931,11 @@ mod tests {
)
.unwrap();
assert_eq!(registry.len(), 1);
+ assert!(registry.is_full());
assert_eq!(registry.retained_preconnect_count(), 0);
let registry = Mutex::new(registry);
- prepare_registry_for_socket(®istry, 1).unwrap();
+ ensure_descriptor_headroom_with_count(®istry, 128, 16, || Ok(32)).unwrap();
assert_eq!(lock(®istry).len(), 1);
}
diff --git a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs
index 486495c956..d8dad148af 100644
--- a/crates/openshell-sandbox/src/sandbox/linux/landlock.rs
+++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs
@@ -149,25 +149,32 @@ fn prepare_capability_free_baseline_at(root: &Path) -> Result {
.into_diagnostic()?
.create()
.into_diagnostic()?;
- let entries = capability_free_baseline_entries(root)?;
- if entries.is_empty() {
- return Err(miette::miette!(
- "capability-free Landlock baseline found no usable root entries"
- ));
- }
- for (_, fd) in entries {
+ let mut usable_entries = 0;
+ // Consume the lazy iterator directly so each path descriptor is added and
+ // dropped before the next root entry is opened. Images can contain
+ // thousands of root entries while the boundary has a 1,024-FD limit.
+ for entry in capability_free_baseline_entries(root)? {
+ let (_, fd) = entry?;
+ usable_entries += 1;
let allowed = access_for_path_fd(&fd, access, abi)?;
ruleset = ruleset
.add_rule(PathBeneath::new(fd, allowed))
.into_diagnostic()?;
}
+ if usable_entries == 0 {
+ return Err(miette::miette!(
+ "capability-free Landlock baseline found no usable root entries"
+ ));
+ }
Ok(PreparedRuleset {
ruleset,
compatibility: LandlockCompatibility::HardRequirement,
})
}
-fn capability_free_baseline_entries(root: &Path) -> Result> {
+fn capability_free_baseline_entries(
+ root: &Path,
+) -> Result>> {
use rustix::fs::{Mode, OFlags, open, openat};
const PRIVATE_ROOT: &str = ".openshell";
@@ -200,27 +207,32 @@ fn capability_free_baseline_entries(root: &Path) -> Result fd,
- Err(rustix::io::Errno::NOENT | rustix::io::Errno::ACCESS) => continue,
- Err(error) => return Err(error).into_diagnostic(),
+ Err(rustix::io::Errno::NOENT | rustix::io::Errno::ACCESS) => return None,
+ Err(error) => return Some(Err(error).into_diagnostic()),
+ };
+ let stat = match rustix::fs::fstat(&fd) {
+ Ok(stat) => stat,
+ Err(error) => return Some(Err(error).into_diagnostic()),
};
- let stat = rustix::fs::fstat(&fd).into_diagnostic()?;
if rustix::fs::FileType::from_raw_mode(stat.st_mode) == rustix::fs::FileType::Symlink {
- continue;
+ return None;
}
- entries.push((entry.path(), fd));
- }
- entries.sort_by(|left, right| left.0.cmp(&right.0));
- Ok(entries)
+ Some(Ok((path, fd)))
+ }))
}
fn prepare_with_path_open_mode(
@@ -627,8 +639,7 @@ mod tests {
.unwrap();
let paths: Vec<_> = capability_free_baseline_entries(root.path())
.unwrap()
- .into_iter()
- .map(|(path, _)| path)
+ .map(|entry| entry.unwrap().0)
.collect();
assert_eq!(
paths,
@@ -652,6 +663,30 @@ mod tests {
assert!(capability_free_baseline_entries(root.path()).is_err());
}
+ #[test]
+ fn capability_free_baseline_drops_each_root_entry_before_opening_the_next() {
+ let root = tempfile::tempdir().unwrap();
+ for index in 0..128 {
+ std::fs::write(root.path().join(format!("entry-{index:03}")), b"public").unwrap();
+ }
+ std::fs::create_dir(root.path().join(".openshell")).unwrap();
+
+ let baseline = std::fs::read_dir("/proc/self/fd").unwrap().count();
+ let mut visited = 0;
+ let mut peak = baseline;
+ for entry in capability_free_baseline_entries(root.path()).unwrap() {
+ let _entry = entry.unwrap();
+ visited += 1;
+ peak = peak.max(std::fs::read_dir("/proc/self/fd").unwrap().count());
+ }
+
+ assert_eq!(visited, 128);
+ assert!(
+ peak <= baseline + 16,
+ "streamed baseline preparation retained too many descriptors: baseline={baseline} peak={peak}"
+ );
+ }
+
#[test]
fn capability_free_baseline_denies_alias_reads_and_path_truncation() {
if !matches!(probe_availability(), LandlockAvailability::Available { abi } if abi >= 3) {
diff --git a/crates/openshell-sdk/tests/client_mock.rs b/crates/openshell-sdk/tests/client_mock.rs
index 9abc5e9e54..9784bea474 100644
--- a/crates/openshell-sdk/tests/client_mock.rs
+++ b/crates/openshell-sdk/tests/client_mock.rs
@@ -685,6 +685,13 @@ impl OpenShell for TestOpenShell {
Ok(Response::new(proto::ProviderResponse::default()))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
_: tonic::Request,
diff --git a/crates/openshell-server/README.md b/crates/openshell-server/README.md
new file mode 100644
index 0000000000..4e35cd7504
--- /dev/null
+++ b/crates/openshell-server/README.md
@@ -0,0 +1,76 @@
+
+
+# OpenShell gateway server
+
+The server owns control-plane RPCs, authentication, provider credential delivery,
+and sandbox lifecycle orchestration. The `openshell-gateway` crate composes the
+server with selected compute drivers.
+
+## Operator credential export
+
+`GetProviderCredentials` exports selected profile-declared runtime environment
+keys from one provider in an explicitly named workspace. It requires the distinct
+`Principal::Operator`, established only from a verified TLS leaf certificate with
+`OU=operator` when `mtls_auth.operator_enabled` is enabled. The TLS listener supplies
+its certificate fingerprint separately from user identity; the ordinary
+identity-only serving path cannot establish an operator.
+
+Operators explicitly inherit user/platform-admin permissions, but not sandbox or
+peer identities. Descriptor authorization and a handler guard both protect the
+export RPC. Operator mutation admission uses a separate caller namespace from
+ordinary mTLS users with the same certificate subject. Credential responses never
+enter durable mutation replay or generic interceptor routing.
+
+The export resolver validates the entire selection before minting, resolves only
+selected credential handles, and fences provider revisions, refresh epochs, and
+profile declarations before delivery. Driver expiration and provider expiration
+combine using the earlier timestamp. OAuth minting inputs, broker-only keys, and
+supervisor-bound dynamic grants are not exportable. A failed response can still
+have committed refresh effects. Missing refresh-owned stored outputs trigger
+refresh-on-read; missing static outputs and other credential-backend errors fail
+closed without falling back to inline values.
+
+## Refresh coordination
+
+Explicit rotation, background refresh, and operator refresh-on-read share the
+same refresh coordination:
+
+- A process-local weak keyed mutex coalesces waiters without retaining deleted
+ providers indefinitely.
+- A bounded admission queue and executor pool limit detached refresh work.
+- PostgreSQL deployments also take a dedicated per-provider/per-credential
+ advisory lock; SQLite uses the local mutex. Worker secret cleanup takes the
+ same locks and reloads state before changing its resource version.
+- A durable `refresh_in_progress` marker precedes upstream minting. A
+ `refresh_committing` marker protects the interval between grant persistence
+ and provider-handle persistence. Only a completed provider commit publishes
+ `refreshed` state, with a unique completed-mint identity in metadata
+ annotations. Waiters coalesce against that identity, not bookkeeping changes
+ to the resource version.
+- Automatic callers recheck live recovery and retry deadlines under both locks.
+ A failed mint blocks queued automatic exchanges without disabling scheduled
+ worker retries or a separately requested manual rotation.
+- Detached refresh tasks complete despite request cancellation, preserving
+ replacement refresh tokens returned by the issuer.
+- Existing generation/CAS fencing remains active for deletion and
+ reconfiguration. An ambiguous rotating-token exchange is parked for recovery
+ rather than automatically consuming the grant again.
+
+A crash or lost database-lock session can leave a parked marker. Reconfigure or
+reauthorize the refresh grant after investigating its issuer state. Changing a
+marker directly is not a supported recovery workflow.
+
+## Focused verification
+
+```shell
+cargo test -p openshell-server --features test-support,prebuilt-z3
+cargo clippy -p openshell-server --all-targets --features test-support,prebuilt-z3 -- -D warnings
+```
+
+The ignored `refresh_locks_coordinate_independent_replica_sessions` test requires
+`OPENSHELL_REFRESH_TEST_DATABASE_URL` pointing to a disposable PostgreSQL database.
+It exercises independent sessions without the local mutex. Do not point this
+fixture at an existing deployment database.
diff --git a/crates/openshell-server/src/auth/descriptor_authz.rs b/crates/openshell-server/src/auth/descriptor_authz.rs
index 7558d14b12..cbdadbdb29 100644
--- a/crates/openshell-server/src/auth/descriptor_authz.rs
+++ b/crates/openshell-server/src/auth/descriptor_authz.rs
@@ -124,6 +124,7 @@ impl DescriptorAuthTable {
"bearer" => AuthMode::Bearer,
"dual" => AuthMode::Dual,
"peer" => AuthMode::Peer,
+ "operator" => AuthMode::Operator,
other => {
return Err(format!("method {path}: unknown auth_mode '{other}'"));
}
diff --git a/crates/openshell-server/src/auth/guard.rs b/crates/openshell-server/src/auth/guard.rs
index a3d55eac71..2ab00a125a 100644
--- a/crates/openshell-server/src/auth/guard.rs
+++ b/crates/openshell-server/src/auth/guard.rs
@@ -29,7 +29,7 @@ use tracing::info;
#[allow(clippy::result_large_err)]
pub fn ensure_sandbox_scope(principal: &Principal, claimed_sandbox_id: &str) -> Result<(), Status> {
match principal {
- Principal::User(_) => Ok(()),
+ Principal::User(_) | Principal::Operator(_) => Ok(()),
Principal::Peer(_) => Err(Status::permission_denied(
"gateway peer principals may not call sandbox-scoped methods",
)),
@@ -87,15 +87,39 @@ pub fn ensure_sandbox_principal_scope(
ensure_sandbox_scope(principal, claimed_sandbox_id)?;
Ok(p.clone())
}
- Principal::User(_) | Principal::Peer(_) => Err(Status::permission_denied(
- "supervisor RPCs require a sandbox principal",
- )),
+ Principal::User(_) | Principal::Operator(_) | Principal::Peer(_) => Err(
+ Status::permission_denied("supervisor RPCs require a sandbox principal"),
+ ),
Principal::Anonymous => Err(Status::unauthenticated(
"supervisor RPCs require an authenticated sandbox principal",
)),
}
}
+/// Require an operator; ordinary admin authority cannot satisfy this guard.
+#[allow(clippy::result_large_err)]
+pub fn require_operator(
+ request: &tonic::Request,
+) -> Result<&super::principal::OperatorPrincipal, Status> {
+ match request.extensions().get::() {
+ Some(Principal::Operator(operator))
+ if operator.identity.provider == super::identity::IdentityProvider::Mtls
+ && operator
+ .identity
+ .roles
+ .iter()
+ .any(|role| role == "operator")
+ && !operator.certificate_sha256.is_empty() =>
+ {
+ Ok(operator)
+ }
+ None | Some(Principal::Anonymous) => {
+ Err(Status::unauthenticated("operator authentication required"))
+ }
+ _ => Err(Status::permission_denied("verified mTLS operator required")),
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/openshell-server/src/auth/method_authz.rs b/crates/openshell-server/src/auth/method_authz.rs
index 06f6784cc8..fb9f3ae461 100644
--- a/crates/openshell-server/src/auth/method_authz.rs
+++ b/crates/openshell-server/src/auth/method_authz.rs
@@ -24,6 +24,8 @@ pub enum AuthMode {
Dual,
/// Only callable by a gateway peer principal.
Peer,
+ /// Only a verified mTLS operator may call this method.
+ Operator,
}
/// Coarse role mapping. Maps to the configured `admin_role` /
@@ -88,11 +90,22 @@ pub fn is_sandbox_callable(method: &str) -> bool {
#[must_use]
pub fn is_user_callable(method: &str) -> bool {
match lookup(method).map(|m| m.auth_mode) {
- Some(AuthMode::Sandbox | AuthMode::Unauthenticated | AuthMode::Peer) => false,
+ Some(
+ AuthMode::Sandbox | AuthMode::Unauthenticated | AuthMode::Peer | AuthMode::Operator,
+ ) => false,
Some(AuthMode::Bearer | AuthMode::Dual) | None => true,
}
}
+/// Operators have user/admin capabilities, but not supervisor or peer identities.
+#[must_use]
+pub fn is_operator_callable(method: &str) -> bool {
+ matches!(
+ lookup(method).map(|entry| entry.auth_mode),
+ Some(AuthMode::Operator | AuthMode::Bearer | AuthMode::Dual)
+ )
+}
+
/// `true` if the method is callable by a gateway peer.
#[must_use]
pub fn is_peer_callable(method: &str) -> bool {
diff --git a/crates/openshell-server/src/auth/principal.rs b/crates/openshell-server/src/auth/principal.rs
index 298533278a..7a4a316d62 100644
--- a/crates/openshell-server/src/auth/principal.rs
+++ b/crates/openshell-server/src/auth/principal.rs
@@ -24,6 +24,8 @@ use super::identity::Identity;
pub enum Principal {
/// Human caller authenticated via OIDC (Keycloak, Entra ID, Okta, etc.).
User(UserPrincipal),
+ /// Gateway-wide administrator authenticated only by a verified operator certificate.
+ Operator(OperatorPrincipal),
/// Sandbox supervisor authenticated by an identity bound to a specific
/// sandbox UUID. The wrapped `sandbox_id` MUST match any sandbox referenced
/// in the request body for sandbox-class methods.
@@ -43,6 +45,26 @@ pub struct UserPrincipal {
pub identity: Identity,
}
+/// Certificate-authenticated gateway operator. Never constructed from bearer claims.
+#[derive(Debug, Clone)]
+pub struct OperatorPrincipal {
+ /// Verified mTLS certificate subject and roles.
+ pub identity: Identity,
+ /// SHA-256 fingerprint of the verified leaf certificate (not a private key).
+ pub certificate_sha256: String,
+}
+
+impl Principal {
+ /// Identity of a human/API caller, preserving its distinct principal variant.
+ pub fn user_identity(&self) -> Option<&Identity> {
+ match self {
+ Self::User(user) => Some(&user.identity),
+ Self::Operator(operator) => Some(&operator.identity),
+ _ => None,
+ }
+ }
+}
+
/// Sandbox caller — bound to one specific sandbox UUID.
///
/// `sandbox_id` and `source` are consumed by the router and handler guards.
diff --git a/crates/openshell-server/src/auth/workspace_authz.rs b/crates/openshell-server/src/auth/workspace_authz.rs
index 0aa38fe0bb..0b8e351b08 100644
--- a/crates/openshell-server/src/auth/workspace_authz.rs
+++ b/crates/openshell-server/src/auth/workspace_authz.rs
@@ -186,6 +186,10 @@ pub async fn authorize_workspace(
grant: AuthGrant::Member(member_role),
})
}
+ Principal::Operator(_) => Ok(AuthorizedWorkspace {
+ workspace,
+ grant: AuthGrant::PlatformAdmin,
+ }),
Principal::Sandbox(_) => Ok(AuthorizedWorkspace {
workspace,
grant: AuthGrant::Sandbox,
@@ -219,6 +223,7 @@ pub async fn authorize_sandbox_workspace(
#[allow(clippy::result_large_err)]
pub fn require_platform_admin(admin_role: &str, principal: &Principal) -> Result<(), Status> {
match principal {
+ Principal::Operator(_) => Ok(()),
Principal::User(user) if is_platform_admin(&user.identity.roles, admin_role) => Ok(()),
Principal::User(_) => Err(Status::permission_denied(
"platform admin role required for cross-workspace operations",
diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs
index bfe237af39..001127761e 100644
--- a/crates/openshell-server/src/cli.rs
+++ b/crates/openshell-server/src/cli.rs
@@ -192,6 +192,10 @@ struct RunArgs {
)]
enable_mtls_auth: bool,
+ /// Enable gateway-wide mTLS operators (trusted certificate OU=operator).
+ #[arg(long, env = "OPENSHELL_ENABLE_OPERATOR_AUTH", default_value_t = false, action = ArgAction::Set)]
+ enable_operator_auth: bool,
+
/// Expected OIDC audience claim (typically the client ID).
#[arg(long, env = "OPENSHELL_OIDC_AUDIENCE", default_value = "openshell-cli")]
oidc_audience: String,
@@ -388,6 +392,11 @@ fn prepare_server_config_with_drivers(
"--disable-tls and --tls-client-ca are mutually exclusive. Client certificate verification requires that TLS be enabled."
));
}
+ if args.enable_operator_auth && (args.disable_tls || !has_client_ca) {
+ return Err(miette::miette!(
+ "operator authentication requires TLS and --tls-client-ca"
+ ));
+ }
if mtls_auth_enabled && args.disable_tls {
return Err(miette::miette!(
"mTLS user authentication requires TLS. Remove --disable-tls or disable --enable-mtls-auth."
@@ -455,6 +464,7 @@ fn prepare_server_config_with_drivers(
config.auth = auth;
}
config.mtls_auth.enabled = mtls_auth_enabled;
+ config.mtls_auth.operator_enabled = args.enable_operator_auth;
// Listener addresses for the health and metrics endpoints. The file may
// pin a different interface than the main listener (e.g. health on
@@ -676,6 +686,11 @@ async fn run_from_args(
if prepared.config.mtls_auth.enabled {
info!("mTLS user authentication enabled");
}
+ if prepared.config.mtls_auth.operator_enabled {
+ info!(
+ "mTLS operator authentication enabled (OU=operator grants gateway-wide administration)"
+ );
+ }
if has_oidc {
info!("OIDC authentication enabled");
}
@@ -699,6 +714,7 @@ async fn run_from_args(
if !prepared.config.auth.allow_unauthenticated_users
&& !prepared.config.mtls_auth.enabled
+ && !prepared.config.mtls_auth.operator_enabled
&& !has_oidc
&& prepared.config.gateway_jwt.is_none()
{
@@ -971,6 +987,11 @@ fn validate_preflight_semantics(
"--disable-tls and --tls-client-ca are mutually exclusive"
));
}
+ if args.enable_operator_auth && (args.disable_tls || !has_client_ca) {
+ return Err(miette::miette!(
+ "operator authentication requires TLS and --tls-client-ca"
+ ));
+ }
if mtls_auth_enabled && args.disable_tls {
return Err(miette::miette!("mTLS user authentication requires TLS"));
}
@@ -1179,6 +1200,11 @@ fn merge_file_into_args(args: &mut RunArgs, file: &GatewayFileSection, matches:
{
args.enable_mtls_auth = mtls_auth.enabled;
}
+ if let Some(mtls_auth) = &file.mtls_auth
+ && arg_defaulted(matches, "enable_operator_auth")
+ {
+ args.enable_operator_auth = mtls_auth.operator_enabled;
+ }
if let Some(disabled) = file.disable_tls
&& arg_defaulted(matches, "disable_tls")
{
@@ -2825,6 +2851,86 @@ enabled = false
));
}
+ #[test]
+ fn operator_auth_is_opt_in_and_file_flag_precedence_is_explicit() {
+ let _lock = ENV_LOCK
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ let _env = EnvVarGuard::remove("OPENSHELL_ENABLE_OPERATOR_AUTH");
+ let file = config_file_from_toml(
+ "[openshell.gateway.mtls_auth]\noperator_enabled = true\nenabled = false\n",
+ );
+ let (mut args, matches) = parse_with_args(&["openshell-gateway"]);
+ assert!(!args.enable_operator_auth);
+ merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
+ assert!(args.enable_operator_auth);
+ assert!(!args.enable_mtls_auth);
+ let (mut args, matches) =
+ parse_with_args(&["openshell-gateway", "--enable-operator-auth", "false"]);
+ merge_file_into_args(&mut args, &file.openshell.gateway, &matches);
+ assert!(!args.enable_operator_auth);
+ }
+
+ #[test]
+ fn operator_preflight_requires_tls_and_ca_independently_of_user_auth() {
+ let _lock = ENV_LOCK
+ .lock()
+ .unwrap_or_else(std::sync::PoisonError::into_inner);
+ let config_home = tempfile::tempdir().unwrap();
+ let _home = EnvVarGuard::set("XDG_CONFIG_HOME", config_home.path().to_str().unwrap());
+ let _config = EnvVarGuard::remove("OPENSHELL_GATEWAY_CONFIG");
+ let _legacy = EnvVarGuard::remove("OPENSHELL_DRIVERS");
+ let _env = EnvVarGuard::remove("OPENSHELL_ENABLE_OPERATOR_AUTH");
+ let registry = test_registry("shared", false);
+ for tls_args in [vec!["--disable-tls"], vec![]] {
+ let mut argv = vec![
+ "openshell-gateway",
+ "--compute-driver",
+ "shared",
+ "--enable-operator-auth",
+ "true",
+ "--enable-mtls-auth",
+ "false",
+ ];
+ argv.extend(tls_args);
+ let (run, matches) = parse_with_args(&argv);
+ let error = super::run_config_preflight_with_drivers(
+ super::ConfigPreflightArgs::default(),
+ run,
+ &matches,
+ ®istry,
+ )
+ .unwrap_err();
+ assert!(
+ error
+ .to_string()
+ .contains("operator authentication requires TLS")
+ );
+ }
+ let (run, matches) = parse_with_args(&[
+ "openshell-gateway",
+ "--compute-driver",
+ "shared",
+ "--enable-operator-auth",
+ "true",
+ "--enable-mtls-auth",
+ "false",
+ "--tls-cert",
+ "/tls/server.pem",
+ "--tls-key",
+ "/tls/key.pem",
+ "--tls-client-ca",
+ "/tls/ca.pem",
+ ]);
+ super::run_config_preflight_with_drivers(
+ super::ConfigPreflightArgs::default(),
+ run,
+ &matches,
+ ®istry,
+ )
+ .unwrap();
+ }
+
#[test]
fn file_value_applies_when_cli_uses_default() {
let _lock = ENV_LOCK
diff --git a/crates/openshell-server/src/grpc/auth_rpc.rs b/crates/openshell-server/src/grpc/auth_rpc.rs
index 848adae6f5..5ba6b4fef7 100644
--- a/crates/openshell-server/src/grpc/auth_rpc.rs
+++ b/crates/openshell-server/src/grpc/auth_rpc.rs
@@ -34,13 +34,12 @@ pub async fn handle_get_current_user(
request: Request,
) -> Result, Status> {
let principal = super::extract_principal(&request)?;
- let Principal::User(user) = principal else {
+ let Some(identity) = principal.user_identity().cloned() else {
return Err(Status::permission_denied(
"GetCurrentUser requires a user principal",
));
};
- let identity = user.identity;
Ok(Response::new(GetCurrentUserResponse {
subject: identity.subject,
display_name: identity.display_name.unwrap_or_default(),
diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs
index 5ac58307a4..fee2c5e8f9 100644
--- a/crates/openshell-server/src/grpc/mod.rs
+++ b/crates/openshell-server/src/grpc/mod.rs
@@ -7,6 +7,7 @@ mod auth_rpc;
pub mod mutation_replay;
pub mod policy;
pub mod provider;
+mod provider_credentials;
pub mod provider_readiness;
mod sandbox;
pub use sandbox::mint_persisted_authentication;
@@ -505,6 +506,13 @@ impl OpenShell for OpenShellService {
mutation_replay::run(&self.state, request).await
}
+ async fn get_provider_credentials(
+ &self,
+ request: Request,
+ ) -> Result, Status> {
+ provider_credentials::handle(&self.state, request).await
+ }
+
async fn get_provider(
&self,
request: Request,
diff --git a/crates/openshell-server/src/grpc/mutation_replay.rs b/crates/openshell-server/src/grpc/mutation_replay.rs
index 19ce936888..a331ea2ec6 100644
--- a/crates/openshell-server/src/grpc/mutation_replay.rs
+++ b/crates/openshell-server/src/grpc/mutation_replay.rs
@@ -150,7 +150,7 @@ pub(super) async fn run(
}
let request_id = validate_request_id(request.get_ref().request_id())?;
let principal = super::extract_principal(&request)?;
- let Principal::User(user) = &principal else {
+ let Some(identity) = principal.user_identity() else {
return Err(Status::permission_denied(
"request admission requires a user principal",
));
@@ -183,7 +183,7 @@ pub(super) async fn run(
} else {
(fingerprint(original)?, None)
};
- let (provider, issuer) = match user.identity.provider {
+ let (provider, issuer) = match identity.provider {
IdentityProvider::Oidc => (
"oidc",
state
@@ -192,15 +192,18 @@ pub(super) async fn run(
.as_ref()
.map_or("", |config| config.issuer.as_str()),
),
- IdentityProvider::Mtls => ("mtls", ""),
+ IdentityProvider::Mtls => (
+ if matches!(principal, Principal::Operator(_)) {
+ "mtls-operator"
+ } else {
+ "mtls"
+ },
+ "",
+ ),
IdentityProvider::CloudflareAccess => ("cloudflare_access", ""),
IdentityProvider::LocalDev => ("local_dev", ""),
};
- let caller = hash_json(&serde_json::json!([
- provider,
- issuer,
- user.identity.subject,
- ]))?;
+ let caller = hash_json(&serde_json::json!([provider, issuer, identity.subject,]))?;
// Not a valid user workspace name; ordinary workspace cleanup cannot touch
// these records. The bucket supports an atomic per-caller storage quota.
let bucket = format!("_mutation/{caller}");
diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs
index 4a0aa0fbc2..8edf83139e 100644
--- a/crates/openshell-server/src/grpc/policy.rs
+++ b/crates/openshell-server/src/grpc/policy.rs
@@ -2688,7 +2688,9 @@ async fn resolve_sandbox_by_name_for_principal(
)?;
Ok(sandbox)
}
- Principal::User(_) => sandbox.ok_or_else(|| Status::not_found("sandbox not found")),
+ Principal::User(_) | Principal::Operator(_) => {
+ sandbox.ok_or_else(|| Status::not_found("sandbox not found"))
+ }
Principal::Peer(_) => Err(Status::permission_denied(
"gateway peer principals may not resolve sandbox configuration",
)),
diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs
index ff8189eab6..9d8b242bd8 100644
--- a/crates/openshell-server/src/grpc/provider.rs
+++ b/crates/openshell-server/src/grpc/provider.rs
@@ -2377,7 +2377,7 @@ fn broker_only_provider_credential_keys_for_provider(
.unwrap_or_default()
}
-fn broker_only_provider_credential_keys(profile: &ProviderProfile) -> HashSet {
+pub(super) fn broker_only_provider_credential_keys(profile: &ProviderProfile) -> HashSet {
profile
.credentials
.iter()
@@ -2404,7 +2404,7 @@ fn provider_credential_not_expired(provider: &Provider, key: &str, now_ms: i64)
.is_none_or(|expiration_ms| expiration_ms > now_ms)
}
-fn is_non_injectable_provider_credential(provider: &Provider, key: &str) -> bool {
+pub(super) fn is_non_injectable_provider_credential(provider: &Provider, key: &str) -> bool {
normalize_profile_id(&provider.r#type).as_deref() == Some("google-vertex-ai")
&& key == "GOOGLE_SERVICE_ACCOUNT_KEY"
}
diff --git a/crates/openshell-server/src/grpc/provider_credentials.rs b/crates/openshell-server/src/grpc/provider_credentials.rs
new file mode 100644
index 0000000000..61666d4d2c
--- /dev/null
+++ b/crates/openshell-server/src/grpc/provider_credentials.rs
@@ -0,0 +1,503 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Privileged export of selected runtime credentials, not raw storage access.
+
+#![allow(clippy::result_large_err)]
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+use std::time::Duration;
+
+use openshell_core::ObjectId;
+use openshell_core::proto::{
+ GetProviderCredentialsRequest, GetProviderCredentialsResponse, Provider,
+ ProviderCredentialRefreshRecoveryAction, ProviderCredentialRefreshStrategy,
+ ProviderCredentialValue,
+};
+use tonic::{Request, Response, Status};
+
+use crate::ServerState;
+use crate::storage_proto::StoredProviderCredentialRefreshStateV2;
+
+#[cfg(test)]
+#[path = "provider_credentials_tests.rs"]
+mod tests;
+
+const MAX_KEYS: usize = 32;
+const MAX_RESPONSE_BYTES: usize = 1024 * 1024;
+const DEFAULT_LIFETIME: Duration = Duration::from_mins(5);
+const MAX_LIFETIME: Duration = Duration::from_hours(24);
+
+#[derive(Default)]
+struct RefreshAudit {
+ attempts: usize,
+ completed: usize,
+}
+
+pub(super) async fn handle(
+ state: &Arc,
+ request: Request,
+) -> Result, Status> {
+ let operator = crate::auth::guard::require_operator(&request)?;
+ let subject = operator.identity.subject.clone();
+ let fingerprint = operator.certificate_sha256.clone();
+ let request = request.into_inner();
+ // Validate before logging selectors: a malformed key could contain a value.
+ let selection =
+ crate::auth::workspace_authz::selected_workspace_name(request.workspace_scope.as_ref())
+ .and_then(|workspace| validate(&request).map(|lifetime| (workspace, lifetime)));
+ let (workspace, lifetime) = match selection {
+ Ok(selection) => selection,
+ Err(error) => {
+ tracing::info!(operator_subject = %subject, certificate_sha256 = %fingerprint,
+ outcome = "invalid_request", status = ?error.code(), "operator provider credential retrieval");
+ return Err(error);
+ }
+ };
+ // Admission survives RPC cancellation, while completion records the refresh
+ // effects even if the final all-or-nothing delivery fails.
+ tracing::info!(
+ operator_subject = %subject,
+ certificate_sha256 = %fingerprint,
+ workspace,
+ provider = %request.name,
+ credential_keys = ?request.credential_keys,
+ outcome = "requested",
+ "operator provider credential retrieval"
+ );
+ let mut refresh_audit = RefreshAudit::default();
+ let result = retrieve(state, workspace, &request, lifetime, &mut refresh_audit).await;
+ tracing::info!(
+ operator_subject = %subject,
+ certificate_sha256 = %fingerprint,
+ workspace,
+ provider = %request.name,
+ credential_keys = ?request.credential_keys,
+ outcome = if result.is_ok() { "delivered" } else { "failed" },
+ refresh_attempts = refresh_audit.attempts,
+ refresh_completed = refresh_audit.completed,
+ "operator provider credential retrieval"
+ );
+ result.map(Response::new)
+}
+
+fn validate(request: &GetProviderCredentialsRequest) -> Result {
+ if request.name.is_empty() || request.name.trim() != request.name || request.name.len() > 253 {
+ return Err(Status::invalid_argument(
+ "canonical provider name is required",
+ ));
+ }
+ if request.credential_keys.is_empty() || request.credential_keys.len() > MAX_KEYS {
+ return Err(Status::invalid_argument(
+ "select between 1 and 32 runtime credential keys",
+ ));
+ }
+ let mut seen = HashSet::new();
+ for key in &request.credential_keys {
+ if key.len() > 256 || !super::provider::is_valid_env_key(key) || !seen.insert(key) {
+ return Err(Status::invalid_argument(
+ "credential keys must be unique runtime environment keys",
+ ));
+ }
+ }
+ let lifetime = request
+ .minimum_remaining_lifetime
+ .as_ref()
+ .map(openshell_core::time::duration_to_std)
+ .transpose()
+ .map_err(|_| Status::invalid_argument("invalid minimum_remaining_lifetime"))?
+ .filter(|duration| !duration.is_zero())
+ .unwrap_or(DEFAULT_LIFETIME);
+ if lifetime > MAX_LIFETIME {
+ return Err(Status::invalid_argument(
+ "minimum_remaining_lifetime exceeds 24 hours",
+ ));
+ }
+ Ok(lifetime)
+}
+
+async fn load(state: &ServerState, workspace: &str, name: &str) -> Result {
+ state
+ .store
+ .get_message_by_name::(workspace, name)
+ .await
+ .map_err(|_| Status::internal("provider lookup failed"))?
+ .ok_or_else(|| Status::not_found("provider not found"))
+}
+
+fn sanitize(error: Status) -> Status {
+ // Driver diagnostics may contain backend details. Never propagate them to
+ // the public export API, or attach them to its audit record.
+ Status::new(
+ error.code(),
+ "runtime credential retrieval failed; inspect provider refresh status",
+ )
+}
+
+async fn retrieve(
+ state: &ServerState,
+ workspace: &str,
+ request: &GetProviderCredentialsRequest,
+ lifetime: Duration,
+ refresh_audit: &mut RefreshAudit,
+) -> Result {
+ let provider = load(state, workspace, &request.name).await?;
+ let catalog = state
+ .provider_profile_sources
+ .snapshot_catalog(state.store.as_ref(), workspace)
+ .await
+ .map_err(sanitize)?;
+ let profile = super::provider::get_provider_type_profile_for_scope(
+ &catalog,
+ &provider.r#type,
+ &provider.profile_workspace,
+ )
+ .ok_or_else(|| {
+ Status::failed_precondition("provider has no supported runtime credential profile")
+ })?
+ .to_proto();
+ let broker_keys = super::provider::broker_only_provider_credential_keys(&profile);
+ // Prevalidate the entire selection before resolving secrets or minting.
+ for key in &request.credential_keys {
+ let declaration = profile
+ .credentials
+ .iter()
+ .find(|credential| credential.env_vars.contains(key));
+ if broker_keys.contains(key)
+ || super::provider::is_non_injectable_provider_credential(&provider, key)
+ || declaration.is_none()
+ || declaration.is_some_and(|credential| credential.token_grant.is_some())
+ {
+ return Err(Status::failed_precondition(
+ "selected key is not an exportable static or gateway-refresh-managed runtime credential",
+ ));
+ }
+ }
+ let refreshes = crate::provider_refresh::list_refresh_states_for_provider(
+ state.store.as_ref(),
+ provider.object_id(),
+ )
+ .await
+ .map_err(sanitize)?;
+ let mut owners = HashMap::new();
+ for key in &request.credential_keys {
+ let matching: Vec<_> = refreshes
+ .iter()
+ .filter(|refresh| {
+ refresh.credential_key == *key
+ || refresh
+ .additional_output_keys
+ .values()
+ .any(|output| output == key)
+ })
+ .collect();
+ if matching.len() > 1 {
+ return Err(Status::failed_precondition(
+ "runtime credential has ambiguous refresh ownership",
+ ));
+ }
+ if let Some(refresh) = matching.first() {
+ if refresh
+ .metadata
+ .as_ref()
+ .is_some_and(|metadata| metadata.deletion_time.is_some())
+ {
+ return Err(Status::failed_precondition(
+ "credential refresh is being deleted",
+ ));
+ }
+ owners.insert(key.clone(), (*refresh).clone());
+ }
+ if !provider.credentials.contains_key(key)
+ && !provider.credential_handles.contains_key(key)
+ && !owners.contains_key(key)
+ {
+ return Err(Status::not_found("selected runtime credential is missing"));
+ }
+ }
+ // State and provider reads are not a transaction. Reload after observing
+ // refresh generations so a just-completed mint is reused, not repeated
+ // against an old provider snapshot paired with a new refresh generation.
+ let initial_provider = load(state, workspace, &request.name).await?;
+ if initial_provider.object_id() != provider.object_id()
+ || initial_provider.r#type != provider.r#type
+ || initial_provider.profile_workspace != provider.profile_workspace
+ {
+ return Err(Status::aborted("provider was replaced or reconfigured"));
+ }
+ // Resolve initial handles individually so NOT_FOUND can be attributed to
+ // a refresh-owned output. A missing static handle or any other backend
+ // error still fails closed, without exporting an inline fallback.
+ let mut initial = HashMap::new();
+ for key in &request.credential_keys {
+ match resolve_selected(state, &initial_provider, std::slice::from_ref(key)).await {
+ Ok(values) => initial.extend(values),
+ Err(error) if error.code() == tonic::Code::NotFound && owners.contains_key(key) => {}
+ Err(error) => return Err(error),
+ }
+ }
+ let mut minted = HashSet::new();
+ for key in &request.credential_keys {
+ if sufficiently_valid(initial.get(key), lifetime)? {
+ continue;
+ }
+ let refresh = owners.get(key).ok_or_else(|| {
+ Status::failed_precondition("credential cannot meet the requested remaining lifetime")
+ })?;
+ ensure_refreshable(refresh)?;
+ if minted.insert(refresh.object_id().to_string()) {
+ refresh_audit.attempts += 1;
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ refresh.clone(),
+ )
+ .await
+ .map_err(sanitize)?;
+ refresh_audit.completed += 1;
+ }
+ }
+ // Capture refresh generations before resolving the final provider snapshot.
+ let mut delivery_owners = HashMap::new();
+ for owner in owners.values() {
+ let latest = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ workspace,
+ provider.object_id(),
+ &owner.credential_key,
+ )
+ .await
+ .map_err(sanitize)?
+ .ok_or_else(|| Status::aborted("credential refresh was deleted"))?;
+ if latest.object_id() != owner.object_id()
+ || crate::provider_refresh::effective_authorization_epoch(&latest)?
+ != crate::provider_refresh::effective_authorization_epoch(owner)?
+ || latest
+ .metadata
+ .as_ref()
+ .is_some_and(|metadata| metadata.deletion_time.is_some())
+ || matches!(
+ latest.status.as_str(),
+ "refresh_in_progress" | "refresh_committing"
+ )
+ {
+ return Err(Status::aborted(
+ "credential refresh changed during retrieval",
+ ));
+ }
+ delivery_owners.insert(owner.credential_key.clone(), latest);
+ }
+ let current = load(state, workspace, &request.name).await?;
+ if current.object_id() != provider.object_id()
+ || current.r#type != provider.r#type
+ || current.profile_workspace != provider.profile_workspace
+ {
+ return Err(Status::aborted("provider was replaced or reconfigured"));
+ }
+ // Fresh handles/expiration come from committed state, never transient minted values.
+ let values = resolve_selected(state, ¤t, &request.credential_keys).await?;
+ for key in &request.credential_keys {
+ if !sufficiently_valid(values.get(key), lifetime)? {
+ return Err(Status::failed_precondition(
+ "credential cannot meet the requested remaining lifetime",
+ ));
+ }
+ }
+ let latest = load(state, workspace, &request.name).await?;
+ if latest
+ .metadata
+ .as_ref()
+ .map(|metadata| (&metadata.id, metadata.resource_version))
+ != current
+ .metadata
+ .as_ref()
+ .map(|metadata| (&metadata.id, metadata.resource_version))
+ {
+ return Err(Status::aborted(
+ "provider changed during credential resolution",
+ ));
+ }
+ // A refresh grant can be reconfigured without a provider update.
+ for owner in delivery_owners.values() {
+ let latest = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ workspace,
+ current.object_id(),
+ &owner.credential_key,
+ )
+ .await
+ .map_err(sanitize)?
+ .ok_or_else(|| Status::aborted("credential refresh was deleted"))?;
+ if latest
+ .metadata
+ .as_ref()
+ .map(|metadata| (&metadata.id, metadata.resource_version))
+ != owner
+ .metadata
+ .as_ref()
+ .map(|metadata| (&metadata.id, metadata.resource_version))
+ || crate::provider_refresh::effective_authorization_epoch(&latest)?
+ != crate::provider_refresh::effective_authorization_epoch(owner)?
+ || latest
+ .metadata
+ .as_ref()
+ .is_some_and(|metadata| metadata.deletion_time.is_some())
+ {
+ return Err(Status::aborted("credential refresh was reconfigured"));
+ }
+ }
+ let latest_catalog = state
+ .provider_profile_sources
+ .snapshot_catalog(state.store.as_ref(), workspace)
+ .await
+ .map_err(sanitize)?;
+ let latest_profile = super::provider::get_provider_type_profile_for_scope(
+ &latest_catalog,
+ ¤t.r#type,
+ ¤t.profile_workspace,
+ )
+ .map(|profile| profile.to_proto());
+ if latest_profile.as_ref() != Some(&profile) {
+ return Err(Status::aborted(
+ "provider credential declarations changed during retrieval",
+ ));
+ }
+ if values
+ .iter()
+ .map(|(key, value)| key.len() + value.value.len() + 64)
+ .sum::()
+ > MAX_RESPONSE_BYTES
+ {
+ return Err(Status::resource_exhausted(
+ "credential response exceeds 1 MiB",
+ ));
+ }
+ // Recheck at delivery, after all driver and database round trips.
+ for value in values.values() {
+ if !sufficiently_valid(Some(value), lifetime)? {
+ return Err(Status::failed_precondition(
+ "credential lifetime elapsed during resolution",
+ ));
+ }
+ }
+ Ok(GetProviderCredentialsResponse {
+ credentials: values,
+ })
+}
+
+fn ensure_refreshable(refresh: &StoredProviderCredentialRefreshStateV2) -> Result<(), Status> {
+ let strategy = ProviderCredentialRefreshStrategy::try_from(refresh.strategy)
+ .unwrap_or(ProviderCredentialRefreshStrategy::Unspecified);
+ if !matches!(
+ strategy,
+ ProviderCredentialRefreshStrategy::Oauth2RefreshToken
+ | ProviderCredentialRefreshStrategy::Oauth2ClientCredentials
+ | ProviderCredentialRefreshStrategy::GoogleServiceAccountJwt
+ | ProviderCredentialRefreshStrategy::AwsStsAssumeRole
+ ) {
+ return Err(Status::failed_precondition(
+ "credential has no gateway-managed minting strategy",
+ ));
+ }
+ // Wait for an active mint through shared coordination. If its owner died,
+ // the durable marker is still rejected after acquiring the distributed lock.
+ if matches!(
+ refresh.status.as_str(),
+ "refresh_in_progress" | "refresh_committing"
+ ) {
+ return Ok(());
+ }
+ if matches!(
+ ProviderCredentialRefreshRecoveryAction::try_from(refresh.recovery_action),
+ Ok(ProviderCredentialRefreshRecoveryAction::Reauthorize
+ | ProviderCredentialRefreshRecoveryAction::FixConfiguration
+ | ProviderCredentialRefreshRecoveryAction::Investigate)
+ ) {
+ return Err(Status::failed_precondition(
+ "credential refresh requires recovery",
+ ));
+ }
+ if refresh.recovery_action == ProviderCredentialRefreshRecoveryAction::Retry as i32
+ && refresh.next_refresh_at_ms > crate::persistence::current_time_ms()
+ {
+ return Err(Status::unavailable(
+ "credential refresh is in retry backoff",
+ ));
+ }
+ Ok(())
+}
+
+async fn resolve_selected(
+ state: &ServerState,
+ provider: &Provider,
+ keys: &[String],
+) -> Result, Status> {
+ let mut selected = provider.clone();
+ selected
+ .credential_handles
+ .retain(|key, _| keys.contains(key));
+ // Expired handle-backed credentials are omitted by the shared runtime and
+ // consequently trigger refresh, or a failure for nonrefreshable credentials.
+ let resolved = state
+ .credentials
+ .resolve_provider_handles(&selected, crate::persistence::current_time_ms())
+ .await
+ .map_err(sanitize)?;
+ let mut result = HashMap::new();
+ for key in keys {
+ if provider.credential_handles.contains_key(key) {
+ if let Some(value) = resolved.values.get(key) {
+ result.insert(
+ key.clone(),
+ ProviderCredentialValue {
+ value: value.clone(),
+ expiration_time: resolved
+ .expires_at_ms
+ .get(key)
+ .map(|expiration| {
+ openshell_core::time::timestamp_from_millis(*expiration)
+ })
+ .transpose()
+ .map_err(|_| {
+ Status::failed_precondition("invalid credential expiration")
+ })?,
+ },
+ );
+ }
+ } else if let Some(value) = provider.credentials.get(key) {
+ result.insert(
+ key.clone(),
+ ProviderCredentialValue {
+ value: value.clone(),
+ expiration_time: provider.credential_expiration_times.get(key).copied(),
+ },
+ );
+ }
+ }
+ Ok(result)
+}
+
+fn sufficiently_valid(
+ value: Option<&ProviderCredentialValue>,
+ lifetime: Duration,
+) -> Result {
+ let Some(value) = value else {
+ return Ok(false);
+ };
+ let Some(expiration) = value.expiration_time.as_ref() else {
+ return Ok(true);
+ };
+ let expiration = openshell_core::time::timestamp_to_millis(expiration)
+ .map_err(|_| Status::failed_precondition("invalid credential expiration"))?;
+ let millis =
+ lifetime.as_millis() + u128::from(!lifetime.subsec_nanos().is_multiple_of(1_000_000));
+ let deadline = crate::persistence::current_time_ms()
+ .checked_add(
+ i64::try_from(millis)
+ .map_err(|_| Status::invalid_argument("invalid remaining lifetime"))?,
+ )
+ .ok_or_else(|| Status::invalid_argument("invalid remaining lifetime"))?;
+ Ok(expiration > deadline)
+}
diff --git a/crates/openshell-server/src/grpc/provider_credentials_tests.rs b/crates/openshell-server/src/grpc/provider_credentials_tests.rs
new file mode 100644
index 0000000000..8e4c6a2fde
--- /dev/null
+++ b/crates/openshell-server/src/grpc/provider_credentials_tests.rs
@@ -0,0 +1,860 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::*;
+use crate::auth::identity::{Identity, IdentityProvider};
+use crate::auth::principal::{OperatorPrincipal, Principal};
+use openshell_core::proto::{
+ ObjectMeta, ProviderProfile, ProviderProfileCredential, workspace_selector,
+};
+use tonic::Code;
+
+fn operator() -> Principal {
+ Principal::Operator(OperatorPrincipal {
+ identity: Identity {
+ subject: "operator-test".into(),
+ display_name: None,
+ roles: vec!["operator".into()],
+ scopes: vec![],
+ provider: IdentityProvider::Mtls,
+ },
+ certificate_sha256: "verified-test-certificate".into(),
+ })
+}
+
+fn request(keys: &[&str]) -> GetProviderCredentialsRequest {
+ GetProviderCredentialsRequest {
+ workspace_scope: Some(workspace_selector("other")),
+ name: "provider".into(),
+ credential_keys: keys.iter().map(|key| (*key).into()).collect(),
+ minimum_remaining_lifetime: None,
+ }
+}
+
+fn authenticated(inner: GetProviderCredentialsRequest) -> Request {
+ let mut request = Request::new(inner);
+ request.extensions_mut().insert(operator());
+ request
+}
+
+async fn fixture() -> (Arc, Provider) {
+ let state = crate::grpc::test_support::test_server_state().await;
+ state
+ .store
+ .put_message(&crate::provider_profile_sources::stored_provider_profile(
+ ProviderProfile {
+ id: "operator-test".into(),
+ credentials: vec![
+ ProviderProfileCredential {
+ name: "access_token".into(),
+ env_vars: vec!["ACCESS_TOKEN".into()],
+ ..Default::default()
+ },
+ ProviderProfileCredential {
+ name: "static_key".into(),
+ env_vars: vec!["STATIC_KEY".into()],
+ ..Default::default()
+ },
+ ],
+ ..Default::default()
+ },
+ ))
+ .await
+ .unwrap();
+ let provider = Provider {
+ metadata: Some(ObjectMeta {
+ id: uuid::Uuid::new_v4().to_string(),
+ name: "provider".into(),
+ workspace: "other".into(),
+ ..Default::default()
+ }),
+ r#type: "operator-test".into(),
+ credentials: HashMap::from([
+ ("ACCESS_TOKEN".into(), "old-access-token".into()),
+ ("STATIC_KEY".into(), "static-secret".into()),
+ ("REFRESH_TOKEN".into(), "must-not-export".into()),
+ ]),
+ ..Default::default()
+ };
+ state.store.put_message(&provider).await.unwrap();
+ (state, provider)
+}
+
+#[tokio::test]
+async fn audits_exports_and_invalid_attempts_without_values() {
+ struct AuditWriter(Arc>>);
+ impl std::io::Write for AuditWriter {
+ fn write(&mut self, bytes: &[u8]) -> std::io::Result {
+ self.0.lock().unwrap().extend_from_slice(bytes);
+ Ok(bytes.len())
+ }
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+ }
+ let (state, _) = fixture().await;
+ let output = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let writer = output.clone();
+ let subscriber = tracing_subscriber::fmt()
+ .without_time()
+ .with_ansi(false)
+ .with_writer(move || AuditWriter(writer.clone()))
+ .finish();
+ let _guard = tracing::subscriber::set_default(subscriber);
+ handle(
+ &state,
+ authenticated(request(&["ACCESS_TOKEN", "STATIC_KEY"])),
+ )
+ .await
+ .unwrap();
+ let error = handle(
+ &state,
+ authenticated(request(&["API_KEY=secret-from-invalid-input"])),
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(error.code(), Code::InvalidArgument);
+ let output = String::from_utf8(output.lock().unwrap().clone()).unwrap();
+ assert!(output.contains("operator-test"));
+ assert!(output.contains("verified-test-certificate"));
+ assert!(output.contains("ACCESS_TOKEN"));
+ assert!(output.contains("STATIC_KEY"));
+ assert!(output.contains("delivered"));
+ assert!(output.contains("invalid_request"));
+ assert!(output.contains("refresh_attempts=0"));
+ for secret in [
+ "old-access-token",
+ "static-secret",
+ "must-not-export",
+ "secret-from-invalid-input",
+ ] {
+ assert!(!output.contains(secret));
+ }
+}
+
+#[tokio::test]
+async fn exports_only_selected_runtime_keys_across_workspaces() {
+ let (state, _) = fixture().await;
+ let response = handle(&state, authenticated(request(&["STATIC_KEY"])))
+ .await
+ .unwrap()
+ .into_inner();
+ assert_eq!(response.credentials.len(), 1);
+ assert_eq!(response.credentials["STATIC_KEY"].value, "static-secret");
+ assert!(response.credentials["STATIC_KEY"].expiration_time.is_none());
+ let error = handle(
+ &state,
+ authenticated(request(&["STATIC_KEY", "REFRESH_TOKEN"])),
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(error.code(), Code::FailedPrecondition);
+ assert!(!error.message().contains("must-not-export"));
+}
+
+#[tokio::test]
+async fn rejects_ordinary_admin_and_anonymous() {
+ let (state, _) = fixture().await;
+ assert_eq!(
+ handle(
+ &state,
+ crate::grpc::test_support::authed_request(request(&["STATIC_KEY"]))
+ )
+ .await
+ .unwrap_err()
+ .code(),
+ Code::PermissionDenied
+ );
+ assert_eq!(
+ handle(&state, Request::new(request(&["STATIC_KEY"])))
+ .await
+ .unwrap_err()
+ .code(),
+ Code::Unauthenticated
+ );
+}
+
+#[test]
+fn validates_selection_and_lifetime() {
+ assert!(validate(&request(&[])).is_err());
+ assert!(validate(&request(&["STATIC_KEY", "STATIC_KEY"])).is_err());
+ assert!(validate(&request(&["../handle"])).is_err());
+ let mut req = request(&["STATIC_KEY"]);
+ req.minimum_remaining_lifetime = Some(prost_types::Duration {
+ seconds: -1,
+ nanos: 0,
+ });
+ assert!(validate(&req).is_err());
+ req.minimum_remaining_lifetime = Some(prost_types::Duration {
+ seconds: 86401,
+ nanos: 0,
+ });
+ assert!(validate(&req).is_err());
+ req.minimum_remaining_lifetime = Some(prost_types::Duration::default());
+ assert_eq!(validate(&req).unwrap(), DEFAULT_LIFETIME);
+}
+
+#[tokio::test]
+async fn lifetime_failure_does_not_return_partial_values() {
+ let (state, mut provider) = fixture().await;
+ provider.credential_expiration_times.insert(
+ "ACCESS_TOKEN".into(),
+ openshell_core::time::timestamp_from_millis(crate::persistence::current_time_ms() + 1000)
+ .unwrap(),
+ );
+ state.store.put_message(&provider).await.unwrap();
+ assert_eq!(
+ handle(
+ &state,
+ authenticated(request(&["STATIC_KEY", "ACCESS_TOKEN"]))
+ )
+ .await
+ .unwrap_err()
+ .code(),
+ Code::FailedPrecondition
+ );
+}
+
+async fn refresh_fixture(
+ state: &ServerState,
+ provider: &Provider,
+ endpoint: String,
+) -> StoredProviderCredentialRefreshStateV2 {
+ let refresh = crate::provider_refresh::new_refresh_state(
+ provider,
+ "other",
+ "ACCESS_TOKEN",
+ crate::provider_refresh::NewRefreshStateConfig {
+ strategy: ProviderCredentialRefreshStrategy::Oauth2RefreshToken,
+ material: HashMap::from([
+ ("refresh_token".into(), "private-refresh-token".into()),
+ ("client_id".into(), "client".into()),
+ ]),
+ secret_material_keys: vec!["refresh_token".into()],
+ expires_at_ms: 1,
+ token_url: endpoint,
+ scopes: vec![],
+ refresh_before: None,
+ max_lifetime: None,
+ additional_output_keys: HashMap::new(),
+ },
+ )
+ .unwrap();
+ crate::provider_refresh::put_refresh_state(state.store.as_ref(), &refresh)
+ .await
+ .unwrap();
+ crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap()
+}
+
+#[tokio::test]
+async fn refreshes_access_token_and_reuses_it() {
+ let (state, mut provider) = fixture().await;
+ provider.credential_expiration_times.insert(
+ "ACCESS_TOKEN".into(),
+ openshell_core::time::timestamp_from_millis(1).unwrap(),
+ );
+ state.store.put_message(&provider).await.unwrap();
+ let issuer = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST")).and(wiremock::matchers::body_string_contains("refresh_token=private-refresh-token"))
+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({"access_token":"new-access-token","refresh_token":"rotated-private-token","expires_in":3600})))
+ .expect(1).mount(&issuer).await;
+ refresh_fixture(&state, &provider, issuer.uri()).await;
+ let (first, concurrent) = tokio::join!(
+ handle(&state, authenticated(request(&["ACCESS_TOKEN"]))),
+ handle(&state, authenticated(request(&["ACCESS_TOKEN"])))
+ );
+ assert_eq!(
+ first.unwrap().into_inner().credentials["ACCESS_TOKEN"].value,
+ "new-access-token"
+ );
+ assert_eq!(
+ concurrent.unwrap().into_inner().credentials["ACCESS_TOKEN"].value,
+ "new-access-token"
+ );
+ for _ in 0..2 {
+ let values = handle(&state, authenticated(request(&["ACCESS_TOKEN"])))
+ .await
+ .unwrap()
+ .into_inner()
+ .credentials;
+ assert_eq!(values["ACCESS_TOKEN"].value, "new-access-token");
+ assert!(values["ACCESS_TOKEN"].expiration_time.is_some());
+ assert_eq!(values.len(), 1);
+ }
+ issuer.verify().await;
+}
+
+#[tokio::test]
+async fn missing_refresh_owned_handle_is_repaired_without_inline_fallback() {
+ let (state, mut provider) = fixture().await;
+ // The stored handle is authoritative, even when a usable-looking inline
+ // value remains. Its absence must mint, never export that inline value.
+ provider.credential_handles.insert(
+ "ACCESS_TOKEN".into(),
+ openshell_core::proto::CredentialHandle {
+ driver: "test-static".into(),
+ handle: "missing-access-token".into(),
+ ..Default::default()
+ },
+ );
+ state.store.put_message(&provider).await.unwrap();
+ let issuer = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .and(wiremock::matchers::body_string_contains(
+ "refresh_token=private-refresh-token",
+ ))
+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(
+ serde_json::json!({"access_token":"repaired-access-token","refresh_token":"replacement-grant","expires_in":3600}),
+ ))
+ .expect(1)
+ .mount(&issuer)
+ .await;
+ refresh_fixture(&state, &provider, issuer.uri()).await;
+ let values = handle(
+ &state,
+ authenticated(request(&["STATIC_KEY", "ACCESS_TOKEN"])),
+ )
+ .await
+ .unwrap()
+ .into_inner()
+ .credentials;
+ assert_eq!(values.len(), 2);
+ assert_eq!(values["STATIC_KEY"].value, "static-secret");
+ assert_eq!(values["ACCESS_TOKEN"].value, "repaired-access-token");
+ assert!(values["ACCESS_TOKEN"].expiration_time.is_some());
+ issuer.verify().await;
+}
+
+#[tokio::test]
+async fn missing_static_handle_and_invalid_refresh_driver_fail_closed() {
+ for (key, driver, expected) in [
+ ("STATIC_KEY", "test-static", Code::NotFound),
+ ("ACCESS_TOKEN", "unconfigured-driver", Code::InvalidArgument),
+ ] {
+ let (state, mut provider) = fixture().await;
+ provider.credential_handles.insert(
+ key.into(),
+ openshell_core::proto::CredentialHandle {
+ driver: driver.into(),
+ handle: "missing-handle".into(),
+ ..Default::default()
+ },
+ );
+ state.store.put_message(&provider).await.unwrap();
+ let issuer = wiremock::MockServer::start().await;
+ refresh_fixture(&state, &provider, issuer.uri()).await;
+ let error = handle(
+ &state,
+ authenticated(request(&["STATIC_KEY", "ACCESS_TOKEN"])),
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(error.code(), expected);
+ assert!(!error.message().contains("missing-handle"));
+ assert!(issuer.received_requests().await.unwrap().is_empty());
+ }
+}
+
+#[tokio::test]
+async fn bookkeeping_changes_do_not_coalesce_refresh_requests() {
+ // Cover both legacy states without a mint identity and states that already
+ // have one. Cleanup/metadata updates must not count as another mint.
+ for previously_minted in [false, true] {
+ let (state, provider) = fixture().await;
+ let issuer = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(
+ serde_json::json!({"access_token":"new-access-token","expires_in":3600}),
+ ))
+ .expect(if previously_minted { 2 } else { 1 })
+ .mount(&issuer)
+ .await;
+ let mut snapshot = refresh_fixture(&state, &provider, issuer.uri()).await;
+ if previously_minted {
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ snapshot,
+ )
+ .await
+ .unwrap();
+ } else {
+ snapshot.status = "refreshed".into();
+ crate::provider_refresh::put_refresh_state(state.store.as_ref(), &snapshot)
+ .await
+ .unwrap();
+ }
+ let snapshot = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ let mut bookkeeping = snapshot.clone();
+ bookkeeping
+ .metadata
+ .as_mut()
+ .unwrap()
+ .labels
+ .insert("bookkeeping".into(), "updated".into());
+ crate::provider_refresh::put_refresh_state(state.store.as_ref(), &bookkeeping)
+ .await
+ .unwrap();
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ snapshot,
+ )
+ .await
+ .unwrap();
+ issuer.verify().await;
+ }
+}
+
+#[tokio::test]
+async fn cancelled_retrieval_finishes_rotating_refresh_token_persistence() {
+ let (state, mut provider) = fixture().await;
+ provider.credential_expiration_times.insert(
+ "ACCESS_TOKEN".into(),
+ openshell_core::time::timestamp_from_millis(1).unwrap(),
+ );
+ state.store.put_message(&provider).await.unwrap();
+ let issuer = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({"access_token":"after-cancel","refresh_token":"rotated-after-cancel","expires_in":3600})).set_delay(Duration::from_millis(200)))
+ .expect(1).mount(&issuer).await;
+ refresh_fixture(&state, &provider, issuer.uri()).await;
+ let caller_state = state.clone();
+ let caller = tokio::spawn(async move {
+ handle(&caller_state, authenticated(request(&["ACCESS_TOKEN"]))).await
+ });
+ tokio::time::timeout(Duration::from_secs(10), async {
+ loop {
+ if !issuer.received_requests().await.unwrap().is_empty() {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ caller.abort();
+ assert!(caller.await.unwrap_err().is_cancelled());
+ tokio::time::timeout(Duration::from_secs(10), async {
+ loop {
+ let refresh = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ if refresh.status == "refreshed" {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ let values = handle(&state, authenticated(request(&["ACCESS_TOKEN"])))
+ .await
+ .unwrap()
+ .into_inner()
+ .credentials;
+ assert_eq!(values["ACCESS_TOKEN"].value, "after-cancel");
+ let refresh = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ let material = state
+ .credentials
+ .resolve_refresh_material(
+ crate::provider_refresh::refresh_material_scope(&refresh),
+ &refresh.secret_material_handles,
+ )
+ .await
+ .unwrap();
+ assert_eq!(material["refresh_token"], "rotated-after-cancel");
+ assert!(!refresh.material.contains_key("refresh_token"));
+ issuer.verify().await;
+}
+
+#[tokio::test]
+async fn coalesces_observed_refresh_generation_and_parks_uncertain_mint() {
+ let (state, provider) = fixture().await;
+ let issuer = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .respond_with(
+ wiremock::ResponseTemplate::new(200)
+ .set_body_json(
+ serde_json::json!({"access_token":"new-access-token","expires_in":3600}),
+ )
+ .set_delay(Duration::from_millis(100)),
+ )
+ .expect(1)
+ .mount(&issuer)
+ .await;
+ let snapshot = refresh_fixture(&state, &provider, issuer.uri()).await;
+ let (a, b) = tokio::join!(
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ snapshot.clone()
+ ),
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ snapshot.clone()
+ )
+ );
+ a.unwrap();
+ b.unwrap();
+ issuer.verify().await;
+ let mut parked = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ parked.status = "refresh_in_progress".into();
+ crate::provider_refresh::put_refresh_state(state.store.as_ref(), &parked)
+ .await
+ .unwrap();
+ let error = crate::provider_refresh::refresh_provider_credential(
+ state.store.as_ref(),
+ "other",
+ &state.credentials,
+ Some(&state.compute),
+ "provider",
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(error.code(), Code::FailedPrecondition);
+ issuer.verify().await;
+}
+
+async fn automatic_waiters_honor_failed_refresh(
+ response: wiremock::ResponseTemplate,
+ expected_code: Code,
+ expected_status: &str,
+ expected_recovery: ProviderCredentialRefreshRecoveryAction,
+) {
+ let (state, provider) = fixture().await;
+ let issuer = wiremock::MockServer::start().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .respond_with(response.set_delay(Duration::from_millis(100)))
+ .expect(1)
+ .mount(&issuer)
+ .await;
+ let snapshot = refresh_fixture(&state, &provider, issuer.uri()).await;
+ let (a, b) = tokio::join!(
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ snapshot.clone(),
+ ),
+ crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ snapshot.clone(),
+ ),
+ );
+ assert_eq!(a.unwrap_err().code(), expected_code);
+ assert_eq!(b.unwrap_err().code(), expected_code);
+ issuer.verify().await;
+ let failed = crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(failed.status, expected_status);
+ assert_eq!(failed.recovery_action, expected_recovery as i32);
+ assert!(!failed.failure_code.is_empty());
+ assert!(failed.next_refresh_at_ms > crate::persistence::current_time_ms());
+ if expected_recovery == ProviderCredentialRefreshRecoveryAction::Reauthorize {
+ assert_eq!(failed.next_refresh_at_ms, i64::MAX);
+ }
+ // Both a stale snapshot and a fresh automatic request must respect the
+ // committed failure, without changing its retry deadline or error receipt.
+ for observed in [snapshot, failed.clone()] {
+ let error = crate::provider_refresh::refresh_from_snapshot(
+ state.store.as_ref(),
+ &state.credentials,
+ Some(&state.compute),
+ observed,
+ )
+ .await
+ .unwrap_err();
+ assert_eq!(error.code(), expected_code);
+ }
+ assert_eq!(
+ crate::provider_refresh::get_refresh_state(
+ state.store.as_ref(),
+ "other",
+ provider.object_id(),
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap(),
+ failed,
+ );
+ issuer.verify().await;
+
+ // A separately requested manual rotation can retry a parked grant or
+ // override retry backoff; it still goes through the same coordination.
+ issuer.reset().await;
+ wiremock::Mock::given(wiremock::matchers::method("POST"))
+ .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(
+ serde_json::json!({"access_token":"manually-refreshed","expires_in":3600}),
+ ))
+ .expect(1)
+ .mount(&issuer)
+ .await;
+ let rotated = crate::provider_refresh::refresh_provider_credential(
+ state.store.as_ref(),
+ "other",
+ &state.credentials,
+ Some(&state.compute),
+ "provider",
+ "ACCESS_TOKEN",
+ )
+ .await
+ .unwrap();
+ assert_eq!(rotated.status, "refreshed");
+ assert_eq!(
+ rotated.recovery_action,
+ ProviderCredentialRefreshRecoveryAction::Unspecified as i32,
+ );
+ issuer.verify().await;
+}
+
+#[tokio::test]
+async fn automatic_waiters_do_not_repeat_unusable_refresh_token_exchange() {
+ automatic_waiters_honor_failed_refresh(
+ wiremock::ResponseTemplate::new(200).set_body_string("not JSON"),
+ Code::FailedPrecondition,
+ "reauthorization_required",
+ ProviderCredentialRefreshRecoveryAction::Reauthorize,
+ )
+ .await;
+}
+
+#[tokio::test]
+async fn automatic_waiters_do_not_bypass_rate_limit_backoff() {
+ automatic_waiters_honor_failed_refresh(
+ wiremock::ResponseTemplate::new(429)
+ .set_body_json(serde_json::json!({"error":"temporarily_unavailable"})),
+ Code::Unavailable,
+ "error",
+ ProviderCredentialRefreshRecoveryAction::Retry,
+ )
+ .await;
+}
+
+#[tokio::test]
+async fn rejects_supervisor_dynamic_grants_even_with_stored_values() {
+ let (state, _) = fixture().await;
+ let mut profile = ProviderProfile {
+ id: "operator-test".into(),
+ credentials: vec![ProviderProfileCredential {
+ name: "dynamic".into(),
+ env_vars: vec!["ACCESS_TOKEN".into()],
+ token_grant: Some(openshell_core::proto::ProviderCredentialTokenGrant::default()),
+ ..Default::default()
+ }],
+ ..Default::default()
+ };
+ // The fixture tests classification, without accepting or executing a grant.
+ profile.source = "user".into();
+ state
+ .store
+ .put_message(&crate::provider_profile_sources::stored_provider_profile(
+ profile,
+ ))
+ .await
+ .unwrap();
+ assert_eq!(
+ handle(&state, authenticated(request(&["ACCESS_TOKEN"])))
+ .await
+ .unwrap_err()
+ .code(),
+ Code::FailedPrecondition
+ );
+}
+
+#[tokio::test]
+async fn real_tls_operator_boundary_and_admin_capabilities() {
+ use crate::multiplex::{
+ MultiplexService, extract_peer_certificate_sha256, extract_peer_identity,
+ };
+ use openshell_core::proto::{
+ GetCurrentUserRequest, ListWorkspacesRequest, open_shell_client::OpenShellClient,
+ };
+ use tonic::transport::{ClientTlsConfig, Endpoint};
+
+ let (mut state, _) = fixture().await;
+ let config = &mut Arc::get_mut(&mut state).unwrap().config;
+ config.mtls_auth.operator_enabled = true;
+ config.mtls_auth.enabled = false;
+ let dir = tempfile::tempdir().unwrap();
+ let (ca, ca_key) = crate::tls_test_utils::generate_test_certs_with_ca(dir.path());
+ let acceptor = crate::TlsAcceptor::from_files(
+ &dir.path().join("server-cert.pem"),
+ &dir.path().join("server-key.pem"),
+ Some(&dir.path().join("ca.pem")),
+ false,
+ None,
+ None,
+ vec![],
+ )
+ .unwrap();
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let service = MultiplexService::new(state);
+ let server = tokio::spawn(async move {
+ loop {
+ let (stream, _) = listener.accept().await.unwrap();
+ openshell_core::net::set_tcp_nodelay_best_effort(&stream);
+ let acceptor = acceptor.acceptor();
+ let service = service.clone();
+ tokio::spawn(async move {
+ if let Ok(stream) = acceptor.accept(stream).await {
+ let identity = extract_peer_identity(&stream);
+ let fingerprint = extract_peer_certificate_sha256(&stream);
+ let _ = service
+ .serve_with_verified_certificate(stream, identity, fingerprint)
+ .await;
+ }
+ });
+ }
+ });
+ for role in [
+ Some("operator"),
+ Some("Operator"),
+ Some("openshell-admin"),
+ None,
+ ] {
+ let mut tls = ClientTlsConfig::new()
+ .ca_certificate(tonic::transport::Certificate::from_pem(ca.pem()))
+ .domain_name("localhost");
+ if let Some(role) = role {
+ let mut params = rcgen::CertificateParams::new(vec![]).unwrap();
+ params
+ .distinguished_name
+ .push(rcgen::DnType::CommonName, "tls-test");
+ params
+ .distinguished_name
+ .push(rcgen::DnType::OrganizationalUnitName, role);
+ params.extended_key_usages = vec![rcgen::ExtendedKeyUsagePurpose::ClientAuth];
+ let key = rcgen::KeyPair::generate().unwrap();
+ let cert = params.signed_by(&key, &ca, &ca_key).unwrap();
+ tls = tls.identity(tonic::transport::Identity::from_pem(
+ cert.pem(),
+ key.serialize_pem(),
+ ));
+ }
+ let channel = Endpoint::from_shared(format!("https://localhost:{}", address.port()))
+ .unwrap()
+ .tls_config(tls)
+ .unwrap()
+ .connect()
+ .await
+ .unwrap();
+ let mut client = OpenShellClient::new(channel);
+ let response = client
+ .get_provider_credentials(request(&["STATIC_KEY"]))
+ .await;
+ if role == Some("operator") {
+ assert_eq!(
+ response.unwrap().into_inner().credentials["STATIC_KEY"].value,
+ "static-secret"
+ );
+ assert_eq!(
+ client
+ .get_current_user(GetCurrentUserRequest {})
+ .await
+ .unwrap()
+ .into_inner()
+ .identity_provider,
+ "mtls"
+ );
+ client
+ .list_workspaces(ListWorkspacesRequest::default())
+ .await
+ .unwrap();
+ let create = openshell_core::proto::CreateWorkspaceRequest {
+ name: "operator-created".into(),
+ request_id: uuid::Uuid::new_v4().to_string(),
+ labels: HashMap::new(),
+ };
+ let first = client
+ .create_workspace(create.clone())
+ .await
+ .unwrap()
+ .into_inner();
+ let replay = client.create_workspace(create).await.unwrap().into_inner();
+ assert_eq!(first, replay);
+ let mut mixed = Request::new(request(&["STATIC_KEY"]));
+ mixed
+ .metadata_mut()
+ .insert("authorization", "Bearer forbidden".parse().unwrap());
+ assert_eq!(
+ client
+ .get_provider_credentials(mixed)
+ .await
+ .unwrap_err()
+ .code(),
+ Code::Unauthenticated
+ );
+ } else {
+ assert_eq!(response.unwrap_err().code(), Code::PermissionDenied);
+ let mut spoof = Request::new(request(&["STATIC_KEY"]));
+ spoof
+ .metadata_mut()
+ .insert("x-forwarded-client-cert", "OU=operator".parse().unwrap());
+ assert_eq!(
+ client
+ .get_provider_credentials(spoof)
+ .await
+ .unwrap_err()
+ .code(),
+ Code::PermissionDenied
+ );
+ }
+ }
+ server.abort();
+}
diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs
index bc432d3582..815fcdd84c 100644
--- a/crates/openshell-server/src/grpc/sandbox.rs
+++ b/crates/openshell-server/src/grpc/sandbox.rs
@@ -322,6 +322,9 @@ pub(super) async fn handle_begin_rootfs_tar_staging(
fn principal_subject(principal: &crate::auth::principal::Principal) -> Result {
match principal {
crate::auth::principal::Principal::User(user) => Ok(user.identity.subject.clone()),
+ crate::auth::principal::Principal::Operator(operator) => {
+ Ok(format!("operator:{}", operator.identity.subject))
+ }
_ => Err(Status::permission_denied(
"rootfs tar staging requires a user principal",
)),
diff --git a/crates/openshell-server/src/grpc/workspace.rs b/crates/openshell-server/src/grpc/workspace.rs
index c8677d890d..8cbc04f527 100644
--- a/crates/openshell-server/src/grpc/workspace.rs
+++ b/crates/openshell-server/src/grpc/workspace.rs
@@ -67,7 +67,7 @@ fn membership_filter_subject<'a>(
Ok(Some(&u.identity.subject))
}
}
- Principal::Sandbox(_) => Ok(None),
+ Principal::Sandbox(_) | Principal::Operator(_) => Ok(None),
Principal::Peer(_) => Err(Status::permission_denied(
"gateway peer principals cannot list workspaces",
)),
diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs
index 8c860ec35c..1de522d257 100644
--- a/crates/openshell-server/src/lib.rs
+++ b/crates/openshell-server/src/lib.rs
@@ -1152,8 +1152,14 @@ fn spawn_gateway_connection(
match acceptor.acceptor().accept(stream).await {
Ok(tls_stream) => {
let peer_identity = multiplex::extract_peer_identity(&tls_stream);
+ let certificate_sha256 =
+ multiplex::extract_peer_certificate_sha256(&tls_stream);
if let Err(e) = service
- .serve_with_peer_identity(tls_stream, peer_identity)
+ .serve_with_verified_certificate(
+ tls_stream,
+ peer_identity,
+ certificate_sha256,
+ )
.await
{
if is_benign_connection_close(e.as_ref()) {
diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs
index 972df8b965..a383f75c4a 100644
--- a/crates/openshell-server/src/multiplex.rs
+++ b/crates/openshell-server/src/multiplex.rs
@@ -47,7 +47,7 @@ use crate::{
auth::authz::AuthzPolicy,
auth::identity::Identity,
auth::oidc::{self, OidcAuthenticator},
- auth::principal::{Principal, UserPrincipal},
+ auth::principal::{OperatorPrincipal, Principal, UserPrincipal},
auth::workspace_authz::{MinWorkspaceRole, authorize_workspace},
http_router, service_http_router,
};
@@ -252,6 +252,35 @@ impl MultiplexService {
where
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
{
+ self.serve_with_verified_certificate(stream, peer_identity, None)
+ .await
+ }
+
+ /// TLS listener only: fingerprint and identity must originate from its verified handshake.
+ pub async fn serve_with_verified_certificate(
+ &self,
+ stream: S,
+ peer_identity: Option,
+ certificate_sha256: Option,
+ ) -> Result<(), Box>
+ where
+ S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
+ {
+ let operator = if self.state.config.mtls_auth.operator_enabled {
+ peer_identity
+ .as_ref()
+ .filter(|identity| {
+ identity.provider == crate::auth::identity::IdentityProvider::Mtls
+ && identity.roles.iter().any(|role| role == "operator")
+ })
+ .zip(certificate_sha256)
+ .map(|(identity, certificate_sha256)| OperatorPrincipal {
+ identity: identity.clone(),
+ certificate_sha256,
+ })
+ } else {
+ None
+ };
let openshell = OpenShellServer::new(OpenShellService::new(self.state.clone()))
.max_decoding_message_size(MAX_GRPC_DECODE_SIZE);
let openshell = GatewayInterceptorGrpcService::new(
@@ -277,7 +306,8 @@ impl MultiplexService {
.flatten(),
self.state.config.mtls_auth.enabled,
self.state.config.auth.allow_unauthenticated_users,
- );
+ )
+ .with_operator(operator);
let grpc_service =
GrpcRateLimitService::new(grpc_service, self.state.grpc_rate_limiter.clone());
let http_service = http_router(self.state.clone());
@@ -685,6 +715,15 @@ fn gateway_principal_fields(principal: &Principal) -> BTreeMap {
fields.insert("scopes".to_string(), user.identity.scopes.join(","));
}
}
+ Principal::Operator(operator) => {
+ fields.insert("kind".to_string(), "operator".to_string());
+ fields.insert("subject".to_string(), operator.identity.subject.clone());
+ fields.insert("provider".to_string(), "mtls".to_string());
+ fields.insert(
+ "certificate_sha256".to_string(),
+ operator.certificate_sha256.clone(),
+ );
+ }
Principal::Sandbox(sandbox) => {
fields.insert("kind".to_string(), "sandbox".to_string());
fields.insert("sandbox_id".to_string(), sandbox.sandbox_id.clone());
@@ -945,6 +984,7 @@ pub struct AuthGrpcRouter {
peer_identity: Option,
mtls_auth_enabled: bool,
allow_unauthenticated_users: bool,
+ operator: Option,
}
impl AuthGrpcRouter {
@@ -972,8 +1012,14 @@ impl AuthGrpcRouter {
peer_identity,
mtls_auth_enabled,
allow_unauthenticated_users,
+ operator: None,
}
}
+
+ fn with_operator(mut self, operator: Option) -> Self {
+ self.operator = operator;
+ self
+ }
}
fn unauthenticated_dev_user_principal() -> Principal {
@@ -1013,6 +1059,7 @@ where
let peer_identity = self.peer_identity.clone();
let mtls_auth_enabled = self.mtls_auth_enabled;
let allow_unauthenticated_users = self.allow_unauthenticated_users;
+ let operator = self.operator.clone();
let mut inner = self.inner.clone();
Box::pin(async move {
@@ -1025,7 +1072,14 @@ where
return inner.ready().await?.call(req).await;
}
- let principal = if let Some(chain) = chain {
+ let principal = if let Some(operator) = operator {
+ if req.headers().contains_key(http::header::AUTHORIZATION) {
+ return Ok(status_response(tonic::Status::unauthenticated(
+ "operator certificates must not be combined with authorization headers",
+ )));
+ }
+ Principal::Operator(operator)
+ } else if let Some(chain) = chain {
match chain.authenticate(req.headers(), &path).await {
Ok(Some(p)) => p,
Ok(None) => match (mtls_auth_enabled, peer_identity) {
@@ -1056,6 +1110,13 @@ where
};
match principal {
+ Principal::Operator(_) => {
+ if !crate::auth::method_authz::is_operator_callable(&path) {
+ return Ok(status_response(tonic::Status::permission_denied(
+ "operators may not call supervisor, peer, or unknown methods",
+ )));
+ }
+ }
Principal::User(ref user) => {
if !crate::auth::method_authz::is_user_callable(&path) {
return Ok(status_response(tonic::Status::permission_denied(
@@ -1287,6 +1348,15 @@ where
})
}
+/// Fingerprint of the leaf certificate verified during the TLS handshake.
+pub fn extract_peer_certificate_sha256(
+ stream: &tokio_rustls::server::TlsStream,
+) -> Option {
+ use sha2::{Digest, Sha256};
+ let certificate = stream.get_ref().1.peer_certificates()?.first()?;
+ Some(format!("{:x}", Sha256::digest(certificate.as_ref())))
+}
+
/// Boxed body type for uniform handling.
pub struct BoxBody(
http_body_util::combinators::UnsyncBoxBody>,
@@ -2730,6 +2800,95 @@ mod tests {
})
}
+ fn operator_principal() -> OperatorPrincipal {
+ let mut identity = mtls_identity("operator");
+ identity.roles = vec!["operator".to_string()];
+ OperatorPrincipal {
+ identity,
+ certificate_sha256: "verified-leaf".to_string(),
+ }
+ }
+
+ #[tokio::test]
+ async fn operator_is_admin_but_not_supervisor_peer_or_unknown() {
+ for (method, permitted) in [
+ ("GetProviderCredentials", true),
+ ("CreateWorkspace", true),
+ ("UpdateProvider", true),
+ ("GetSandboxConfig", true),
+ ("ConnectSupervisor", false),
+ ("PeerRelay", false),
+ ("IssueSandboxToken", false),
+ ("Unknown", false),
+ ] {
+ let (recorder, seen) = PrincipalRecorder::new();
+ let mut router = AuthGrpcRouter::with_peer_identity(
+ recorder,
+ None,
+ Some(AuthzPolicy {
+ admin_role: "oidc-admin".into(),
+ user_role: "oidc-user".into(),
+ scopes_enabled: true,
+ }),
+ None,
+ false,
+ false,
+ )
+ .with_operator(Some(operator_principal()));
+ let response = router
+ .call(empty_request(&format!("/openshell.v1.OpenShell/{method}")))
+ .await
+ .unwrap();
+ if permitted {
+ assert!(grpc_status(&response).is_none());
+ assert!(matches!(
+ *seen.lock().unwrap(),
+ Some(Principal::Operator(_))
+ ));
+ } else {
+ assert_eq!(grpc_status(&response).as_deref(), Some("7"));
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn user_bearer_claims_and_local_dev_cannot_export_credentials() {
+ let mut user = mtls_identity("ordinary-user");
+ user.provider = IdentityProvider::Oidc;
+ user.roles = vec!["operator".into(), "openshell-admin".into()];
+ user.scopes = vec!["openshell:all".into()];
+ for principal in [
+ Principal::User(UserPrincipal { identity: user }),
+ sandbox_principal(),
+ Principal::Peer(crate::auth::principal::PeerPrincipal {
+ replica_id: "replica".into(),
+ pod_uid: "pod".into(),
+ }),
+ ] {
+ let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(
+ Ok(Some(principal)),
+ ))]);
+ let (recorder, _) = PrincipalRecorder::new();
+ let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
+ let response = router
+ .call(empty_request(
+ "/openshell.v1.OpenShell/GetProviderCredentials",
+ ))
+ .await
+ .unwrap();
+ assert_eq!(grpc_status(&response).as_deref(), Some("7"));
+ }
+ let (recorder, _) = PrincipalRecorder::new();
+ let mut router = AuthGrpcRouter::new(recorder, None, None);
+ let response = router
+ .call(empty_request(
+ "/openshell.v1.OpenShell/GetProviderCredentials",
+ ))
+ .await
+ .unwrap();
+ assert_eq!(grpc_status(&response).as_deref(), Some("7"));
+ }
+
#[derive(Serialize)]
struct LegacySandboxClaims {
sub: String,
diff --git a/crates/openshell-server/src/persistence/mod.rs b/crates/openshell-server/src/persistence/mod.rs
index 1721cf7931..81c939faad 100644
--- a/crates/openshell-server/src/persistence/mod.rs
+++ b/crates/openshell-server/src/persistence/mod.rs
@@ -5,6 +5,8 @@
mod legacy_time_wire;
mod postgres;
+#[cfg(test)]
+mod refresh_lock_tests;
mod sqlite;
pub use crate::storage_proto::{
@@ -312,6 +314,29 @@ impl Store {
}
}
+ /// Independent per-credential lock, shared across `PostgreSQL` replicas.
+ /// Callers also hold a local keyed mutex on `SQLite`.
+ pub(crate) async fn acquire_refresh_guard(
+ &self,
+ provider_id: &str,
+ credential_key: &str,
+ ) -> PersistenceResult {
+ use sha2::{Digest, Sha256};
+ let mut digest = Sha256::new();
+ digest.update(b"openshell-provider-refresh-lock-v1\0");
+ digest.update(provider_id.as_bytes());
+ digest.update(b"\0");
+ digest.update(credential_key.as_bytes());
+ let hash = digest.finalize();
+ let key = i64::from_be_bytes(hash[..8].try_into().expect("eight digest bytes"));
+ match self {
+ Self::Postgres(store) => Ok(DistributedMutationGuard {
+ _postgres: Some(store.acquire_mutation_lock(key).await?),
+ }),
+ Self::Sqlite(_) => Ok(DistributedMutationGuard { _postgres: None }),
+ }
+ }
+
/// Connect to a persistence store based on the database URL.
pub async fn connect(url: &str) -> CoreResult {
if url.starts_with("postgres://") || url.starts_with("postgresql://") {
diff --git a/crates/openshell-server/src/persistence/refresh_lock_tests.rs b/crates/openshell-server/src/persistence/refresh_lock_tests.rs
new file mode 100644
index 0000000000..21791085c4
--- /dev/null
+++ b/crates/openshell-server/src/persistence/refresh_lock_tests.rs
@@ -0,0 +1,48 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use super::Store;
+use std::time::Duration;
+
+#[tokio::test]
+#[ignore = "requires a disposable PostgreSQL database in OPENSHELL_REFRESH_TEST_DATABASE_URL"]
+async fn refresh_locks_coordinate_independent_replica_sessions() {
+ let url =
+ std::env::var("OPENSHELL_REFRESH_TEST_DATABASE_URL").expect("disposable database URL");
+ let first = Store::connect(&url).await.unwrap();
+ let second = Store::connect(&url).await.unwrap();
+ let provider = uuid::Uuid::new_v4().to_string();
+ let guard = first
+ .acquire_refresh_guard(&provider, "ACCESS_TOKEN")
+ .await
+ .unwrap();
+ // This bypasses the process-local mutex: independent database sessions must
+ // still serialize a mint, while unrelated credentials remain independent.
+ assert!(
+ tokio::time::timeout(
+ Duration::from_millis(100),
+ second.acquire_refresh_guard(&provider, "ACCESS_TOKEN")
+ )
+ .await
+ .is_err()
+ );
+ let unrelated = tokio::time::timeout(
+ Duration::from_secs(2),
+ second.acquire_refresh_guard(&provider, "OTHER_KEY"),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ drop(unrelated);
+ drop(guard);
+ let guard = tokio::time::timeout(
+ Duration::from_secs(2),
+ second.acquire_refresh_guard(&provider, "ACCESS_TOKEN"),
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ drop(guard);
+ first.close().await;
+ second.close().await;
+}
diff --git a/crates/openshell-server/src/provider_refresh.rs b/crates/openshell-server/src/provider_refresh.rs
index c1b5184d18..7708022f0b 100644
--- a/crates/openshell-server/src/provider_refresh.rs
+++ b/crates/openshell-server/src/provider_refresh.rs
@@ -29,6 +29,7 @@ use crate::storage_proto::{
const DEFAULT_REFRESH_BEFORE_SECONDS: i64 = 300;
const EXPIRATION_PRESENT_ANNOTATION: &str = "openshell.nvidia.com/refresh-expiration-present";
+const COMPLETED_MINT_ANNOTATION: &str = "openshell.nvidia.com/refresh-completed-mint";
const DEFAULT_MAX_LIFETIME_SECONDS: i64 = 3600;
const REFRESH_ERROR_RETRY_SECONDS: i64 = 60;
const REFRESH_CONFIGURATION_RETRY_SECONDS: i64 = 60 * 60;
@@ -845,6 +846,115 @@ fn validate_secret_material_references(
)))
}
+// Local waiters do not consume PostgreSQL sessions while another local mint owns
+// the credential. Weak entries avoid retaining every provider ever refreshed.
+static REFRESH_LOCKS: std::sync::LazyLock<
+ std::sync::Mutex>>>,
+> = std::sync::LazyLock::new(|| std::sync::Mutex::new(HashMap::new()));
+static REFRESH_EXECUTORS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(4);
+static REFRESH_ADMISSIONS: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(32);
+
+fn refresh_lock(
+ state: &StoredProviderCredentialRefreshState,
+) -> Result>, Status> {
+ let key = refresh_state_name(&state.provider_id, &state.credential_key);
+ let mut locks = REFRESH_LOCKS
+ .lock()
+ .map_err(|_| Status::internal("refresh coordination unavailable"))?;
+ locks.retain(|_, lock| lock.strong_count() > 0);
+ let lock = locks
+ .get(&key)
+ .and_then(std::sync::Weak::upgrade)
+ .unwrap_or_else(|| std::sync::Arc::new(tokio::sync::Mutex::new(())));
+ locks.insert(key, std::sync::Arc::downgrade(&lock));
+ Ok(lock)
+}
+
+fn completed_mint(state: &StoredProviderCredentialRefreshState) -> Option<&str> {
+ state
+ .metadata
+ .as_ref()?
+ .annotations
+ .get(COMPLETED_MINT_ANNOTATION)
+ .map(String::as_str)
+ .filter(|id| !id.is_empty())
+}
+
+/// Cleanup also mutates the mint receipt's resource version. Use the same
+/// local/distributed locks as minting, then reload instead of writing a stale
+/// sweep snapshot over an active mint or deleting its replacement grant.
+async fn cleanup_pending_secret_deletions_coordinated(
+ store: &Store,
+ credentials: Option<&crate::credentials::CredentialRuntime>,
+ state: &mut StoredProviderCredentialRefreshState,
+) -> Result<(), Status> {
+ if state.pending_secret_deletions.is_empty() {
+ return Ok(());
+ }
+ let _local = tokio::time::timeout(Duration::from_secs(40), refresh_lock(state)?.lock_owned())
+ .await
+ .map_err(|_| Status::unavailable("credential refresh is busy"))?;
+ let _distributed = store
+ .acquire_refresh_guard(&state.provider_id, &state.credential_key)
+ .await
+ .map_err(|_| Status::unavailable("credential refresh coordination unavailable"))?;
+ let latest = get_refresh_state(
+ store,
+ state.object_workspace(),
+ &state.provider_id,
+ &state.credential_key,
+ )
+ .await?
+ .ok_or_else(|| Status::aborted("provider refresh was deleted"))?;
+ if latest.object_id() != state.object_id()
+ || effective_authorization_epoch(&latest)? != effective_authorization_epoch(state)?
+ || latest
+ .metadata
+ .as_ref()
+ .is_some_and(|metadata| metadata.deletion_time.is_some())
+ {
+ return Err(Status::aborted(
+ "provider refresh was deleted or reconfigured",
+ ));
+ }
+ *state = latest;
+ let expected_version = state
+ .metadata
+ .as_ref()
+ .map_or(0, |metadata| metadata.resource_version);
+ cleanup_pending_secret_deletions(store, credentials, state, expected_version).await?;
+ Ok(())
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum RefreshTrigger {
+ Automatic,
+ ExplicitRotation,
+}
+
+/// Automatic callers must honor the live failure schedule, not the eligibility
+/// observed before they queued. Configuration/investigation failures can still
+/// be retried by the worker once due; reauthorization stays parked.
+fn ensure_automatic_refresh_ready(
+ state: &StoredProviderCredentialRefreshState,
+ now_ms: i64,
+) -> Result<(), Status> {
+ if state.recovery_action == ProviderCredentialRefreshRecoveryAction::Reauthorize as i32 {
+ return Err(Status::failed_precondition(
+ "credential refresh requires recovery",
+ ));
+ }
+ if state.recovery_action != ProviderCredentialRefreshRecoveryAction::Unspecified as i32
+ && state.next_refresh_at_ms > now_ms
+ {
+ return Err(Status::unavailable(
+ "credential refresh is in retry backoff",
+ ));
+ }
+ Ok(())
+}
+
+/// Cancellation-safe entry point for a separately requested manual rotation.
pub async fn refresh_provider_credential(
store: &Store,
workspace: &str,
@@ -852,6 +962,142 @@ pub async fn refresh_provider_credential(
compute: Option<&crate::compute::ComputeRuntime>,
provider_name: &str,
credential_key: &str,
+) -> Result {
+ let provider = store
+ .get_message_by_name::(workspace, provider_name)
+ .await
+ .map_err(|_| Status::internal("fetch provider failed"))?
+ .ok_or_else(|| Status::not_found("provider not found"))?;
+ let snapshot = get_refresh_state(store, workspace, provider.object_id(), credential_key)
+ .await?
+ .ok_or_else(|| Status::not_found("provider refresh state not found"))?;
+ refresh_from_snapshot_with_trigger(
+ store,
+ credentials,
+ compute,
+ snapshot,
+ RefreshTrigger::ExplicitRotation,
+ )
+ .await
+}
+
+/// Coalesce an automatic refresh against the caller's observed generation, not
+/// a name-keyed replacement. Recheck recovery/backoff after acquiring the locks.
+pub async fn refresh_from_snapshot(
+ store: &Store,
+ credentials: &crate::credentials::CredentialRuntime,
+ compute: Option<&crate::compute::ComputeRuntime>,
+ snapshot: StoredProviderCredentialRefreshState,
+) -> Result {
+ refresh_from_snapshot_with_trigger(
+ store,
+ credentials,
+ compute,
+ snapshot,
+ RefreshTrigger::Automatic,
+ )
+ .await
+}
+
+async fn refresh_from_snapshot_with_trigger(
+ store: &Store,
+ credentials: &crate::credentials::CredentialRuntime,
+ compute: Option<&crate::compute::ComputeRuntime>,
+ snapshot: StoredProviderCredentialRefreshState,
+ trigger: RefreshTrigger,
+) -> Result {
+ let admission = REFRESH_ADMISSIONS
+ .try_acquire()
+ .map_err(|_| Status::resource_exhausted("credential refresh queue is full"))?;
+ let lock = refresh_lock(&snapshot)?;
+ let store = store.clone();
+ let credentials = credentials.clone();
+ let compute = compute.cloned();
+ // A disconnected RPC must not cancel minting after an upstream rotates its
+ // refresh token. Spawn before any externally observable refresh side effect.
+ tokio::spawn(async move {
+ let _admission = admission;
+ let _local = tokio::time::timeout(Duration::from_secs(40), lock.lock_owned())
+ .await
+ .map_err(|_| Status::unavailable("credential refresh is busy"))?;
+ let _permit = tokio::time::timeout(Duration::from_secs(40), REFRESH_EXECUTORS.acquire())
+ .await
+ .map_err(|_| Status::resource_exhausted("credential refresh workers are busy"))?
+ .map_err(|_| Status::unavailable("credential refresh workers stopped"))?;
+ let _distributed = store
+ .acquire_refresh_guard(&snapshot.provider_id, &snapshot.credential_key)
+ .await
+ .map_err(|_| Status::unavailable("credential refresh coordination unavailable"))?;
+ let latest = get_refresh_state(
+ &store,
+ snapshot.object_workspace(),
+ &snapshot.provider_id,
+ &snapshot.credential_key,
+ )
+ .await?
+ .ok_or_else(|| Status::aborted("provider refresh was deleted"))?;
+ if latest.object_id() != snapshot.object_id()
+ || effective_authorization_epoch(&latest)? != effective_authorization_epoch(&snapshot)?
+ {
+ return Err(Status::aborted("provider refresh was reconfigured"));
+ }
+ if latest
+ .metadata
+ .as_ref()
+ .is_some_and(|metadata| metadata.deletion_time.is_some())
+ {
+ return Err(Status::failed_precondition(
+ "provider refresh is being deleted",
+ ));
+ }
+ // A row-version change can be cleanup or other bookkeeping, not a
+ // successful exchange. Only a new committed mint can satisfy a waiter.
+ if latest.status == "refreshed"
+ && completed_mint(&latest).is_some()
+ && completed_mint(&latest) != completed_mint(&snapshot)
+ {
+ return Ok(latest);
+ }
+ let provider = store
+ .get_message_by_name::(snapshot.object_workspace(), &snapshot.provider_name)
+ .await
+ .map_err(|_| Status::internal("fetch provider failed"))?
+ .ok_or_else(|| Status::not_found("provider not found"))?;
+ if provider.object_id() != snapshot.provider_id {
+ return Err(Status::aborted("provider was replaced"));
+ }
+ if matches!(
+ latest.status.as_str(),
+ "refresh_in_progress" | "refresh_committing"
+ ) {
+ return Err(Status::failed_precondition(
+ "previous credential mint has an uncertain outcome; reconfigure the refresh grant",
+ ));
+ }
+ if trigger == RefreshTrigger::Automatic {
+ ensure_automatic_refresh_ready(&latest, current_time_ms())?;
+ }
+ refresh_provider_credential_locked(
+ &store,
+ snapshot.object_workspace(),
+ &credentials,
+ compute.as_ref(),
+ &snapshot.provider_name,
+ &snapshot.credential_key,
+ )
+ .await
+ })
+ .await
+ .map_err(|_| Status::internal("credential refresh worker failed"))?
+}
+
+async fn refresh_provider_credential_locked(
+ store: &Store,
+ workspace: &str,
+ credentials: &crate::credentials::CredentialRuntime,
+ compute: Option<&crate::compute::ComputeRuntime>,
+ provider_name: &str,
+ credential_key: &str,
) -> Result {
let provider = store
.get_message_by_name::(workspace, provider_name)
@@ -911,6 +1157,14 @@ pub async fn refresh_provider_credential(
"provider credential refresh started"
);
+ // Persist uncertainty BEFORE contacting the issuer. A crashed process or a
+ // lost advisory-lock session cannot lead another replica to reuse the grant.
+ state.status = "refresh_in_progress".to_string();
+ state.next_refresh_at_ms = i64::MAX;
+ state.recovery_action = ProviderCredentialRefreshRecoveryAction::Investigate as i32;
+ let expected_version = persist_refresh_state_if_current(store, &state, expected_version)
+ .await?
+ .ok_or_else(|| Status::aborted("provider refresh changed before minting"))?;
let mint_result = match resolve_refresh_material(Some(credentials), &state).await {
Ok(transient_state) => mint_credential(&transient_state).await,
Err(err) => Err(err.into()),
@@ -995,7 +1249,9 @@ pub async fn refresh_provider_credential(
state.next_refresh_at_ms =
next_refresh_at_ms(minted.expires_at_ms, state.refresh_before.as_ref());
state.last_refresh_at_ms = now_ms;
- state.status = "refreshed".to_string();
+ let completed_next_refresh_at_ms = state.next_refresh_at_ms;
+ state.status = "refresh_committing".to_string();
+ state.next_refresh_at_ms = i64::MAX;
state.last_error.clear();
state.recovery_action = ProviderCredentialRefreshRecoveryAction::Unspecified as i32;
state.failure_code.clear();
@@ -1083,6 +1339,22 @@ pub async fn refresh_provider_credential(
);
return Err(err);
}
+ // A success snapshot must not become visible before its provider
+ // handles are committed. Park incomplete commits rather than minting
+ // again after a crash or lost database session.
+ state.status = "refreshed".to_string();
+ state.next_refresh_at_ms = completed_next_refresh_at_ms;
+ if let Some(metadata) = state.metadata.as_mut() {
+ metadata.annotations.insert(
+ COMPLETED_MINT_ANNOTATION.to_string(),
+ uuid::Uuid::new_v4().to_string(),
+ );
+ }
+ let new_version = persist_refresh_state_if_current(store, &state, new_version)
+ .await?
+ .ok_or_else(|| {
+ Status::aborted("provider refresh changed during credential commit")
+ })?;
info!(
provider = %state.provider_name,
credential_key = %state.credential_key,
@@ -1641,10 +1913,20 @@ async fn request_token(
error_kind,
"OAuth token endpoint request failed"
);
- RefreshFailure::retryable(
- Status::unavailable("token endpoint request failed"),
- "oauth_token_endpoint_unavailable",
- )
+ if grant_kind == OAuthGrantKind::UserRefreshToken && error_kind != "connect" {
+ RefreshFailure::reauthorize(
+ Status::failed_precondition(
+ "refresh token exchange has an uncertain outcome; reauthorize the grant",
+ ),
+ "oauth_refresh_exchange_uncertain",
+ None,
+ )
+ } else {
+ RefreshFailure::retryable(
+ Status::unavailable("token endpoint request failed"),
+ "oauth_token_endpoint_unavailable",
+ )
+ }
})?;
let status = response.status();
if !status.is_success() {
@@ -1652,12 +1934,30 @@ async fn request_token(
return Err(classify_oauth_token_error(status, &body, grant_kind));
}
let token = response.json::().await.map_err(|_| {
+ if grant_kind == OAuthGrantKind::UserRefreshToken {
+ return RefreshFailure::reauthorize(
+ Status::failed_precondition(
+ "refresh token exchange returned an unusable response; reauthorize the grant",
+ ),
+ "oauth_refresh_exchange_uncertain",
+ None,
+ );
+ }
RefreshFailure::investigate(
Status::failed_precondition("token endpoint returned invalid JSON"),
"oauth_invalid_success_response",
)
})?;
if token.access_token.trim().is_empty() {
+ if grant_kind == OAuthGrantKind::UserRefreshToken {
+ return Err(RefreshFailure::reauthorize(
+ Status::failed_precondition(
+ "refresh token exchange returned no access token; reauthorize the grant",
+ ),
+ "oauth_refresh_exchange_uncertain",
+ None,
+ ));
+ }
return Err(RefreshFailure::investigate(
Status::failed_precondition("token endpoint returned empty access_token"),
"oauth_empty_access_token",
@@ -2018,12 +2318,8 @@ async fn refresh_states(
continue;
}
let mut state = state;
- let expected_version = state
- .metadata
- .as_ref()
- .map_or(0, |metadata| metadata.resource_version);
if let Err(err) =
- cleanup_pending_secret_deletions(store, credentials, &mut state, expected_version).await
+ cleanup_pending_secret_deletions_coordinated(store, credentials, &mut state).await
{
warn!(
provider = %state.provider_name,
@@ -2078,15 +2374,14 @@ async fn refresh_states(
status = %state.status,
"refreshing provider credential"
);
- if let Err(err) = refresh_provider_credential(
- store,
- state.object_workspace(),
- credentials,
- compute,
- &state.provider_name,
- &state.credential_key,
- )
- .await
+ let trigger = if rotation_requested {
+ RefreshTrigger::ExplicitRotation
+ } else {
+ RefreshTrigger::Automatic
+ };
+ if let Err(err) =
+ refresh_from_snapshot_with_trigger(store, credentials, compute, state.clone(), trigger)
+ .await
{
warn!(
provider = %state.provider_name,
@@ -2107,9 +2402,9 @@ mod tests {
MAX_OAUTH_ERROR_RESPONSE_BYTES, NewRefreshStateConfig, OAuthGrantKind, RefreshFailure,
RefreshRetrySchedule, Status, classify_oauth_token_error,
delete_refresh_state_with_credentials, effective_authorization_epoch,
- enqueue_pending_secret_deletion, get_refresh_state, list_all_refresh_states,
- list_refresh_states_for_provider, max_lifetime_seconds, new_refresh_state,
- next_refresh_at_ms, put_refresh_state, read_bounded_oauth_error_body,
+ enqueue_pending_secret_deletion, ensure_automatic_refresh_ready, get_refresh_state,
+ list_all_refresh_states, list_refresh_states_for_provider, max_lifetime_seconds,
+ new_refresh_state, next_refresh_at_ms, put_refresh_state, read_bounded_oauth_error_body,
refresh_has_expiration, refresh_material_scope, refresh_provider_credential,
refresh_state_name, refresh_status_from_state, refresh_strategy_name,
run_refresh_worker_tick, seconds_until_ms, set_refresh_expiration_presence,
@@ -3236,6 +3531,158 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn worker_cleanup_waits_for_mint_and_preserves_rotated_grant() {
+ use std::time::Duration;
+
+ let issuer = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(body_string_contains("refresh_token=old-grant"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
+ "access_token": "new-access-token",
+ "refresh_token": "replacement-grant",
+ "expires_in": 3600
+ })))
+ .expect(1)
+ .mount(&issuer)
+ .await;
+ let store = test_store().await;
+ let provider = provider("cleanup-during-mint", "outlook");
+ store.put_message(&provider).await.unwrap();
+ let credentials = test_credentials();
+ let mut state = new_refresh_state(
+ &provider,
+ "default",
+ "MS_GRAPH_ACCESS_TOKEN",
+ NewRefreshStateConfig {
+ strategy: ProviderCredentialRefreshStrategy::Oauth2RefreshToken,
+ material: HashMap::from([
+ ("client_id".into(), "client".into()),
+ ("refresh_token".into(), "old-grant".into()),
+ ]),
+ secret_material_keys: vec!["refresh_token".into()],
+ expires_at_ms: 1,
+ token_url: issuer.uri(),
+ scopes: vec![],
+ refresh_before: None,
+ max_lifetime: None,
+ additional_output_keys: HashMap::new(),
+ },
+ )
+ .unwrap();
+ let obsolete = credentials
+ .store_refresh_material_with_object_id(
+ refresh_material_scope(&state),
+ "obsolete-grant",
+ &HashMap::from([("refresh_token".into(), "obsolete".into())]),
+ &HashMap::new(),
+ )
+ .await
+ .unwrap()
+ .remove("refresh_token")
+ .unwrap();
+ enqueue_pending_secret_deletion(&mut state, "refresh_token", obsolete);
+ put_refresh_state(&store, &state).await.unwrap();
+ let sweep_snapshot = get_refresh_state(
+ &store,
+ "default",
+ provider.object_id(),
+ "MS_GRAPH_ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ // Fail pre-mint cleanup, then stop at replacement-grant storage after
+ // the issuer has rotated the grant but before the receipt CAS.
+ credentials.fail_next_delete();
+ let (storing, release) = credentials.gate_next_store();
+ let mint_store = store.clone();
+ let mint_credentials = credentials.clone();
+ let mint = tokio::spawn(async move {
+ refresh_provider_credential(
+ &mint_store,
+ "default",
+ &mint_credentials,
+ None,
+ "cleanup-during-mint",
+ "MS_GRAPH_ACCESS_TOKEN",
+ )
+ .await
+ });
+ tokio::time::timeout(Duration::from_secs(10), storing)
+ .await
+ .unwrap()
+ .unwrap();
+ let before_cleanup = get_refresh_state(
+ &store,
+ "default",
+ provider.object_id(),
+ "MS_GRAPH_ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(before_cleanup.status, "refresh_in_progress");
+ let cleanup_store = store.clone();
+ let cleanup_credentials = credentials.clone();
+ let mut sweep = tokio::spawn(async move {
+ super::refresh_states(
+ &cleanup_store,
+ Some(&cleanup_credentials),
+ None,
+ vec![sweep_snapshot],
+ current_time_ms(),
+ )
+ .await;
+ });
+ assert!(
+ tokio::time::timeout(Duration::from_millis(100), &mut sweep)
+ .await
+ .is_err(),
+ "cleanup must wait for the active mint"
+ );
+ let during_cleanup = get_refresh_state(
+ &store,
+ "default",
+ provider.object_id(),
+ "MS_GRAPH_ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(during_cleanup, before_cleanup);
+ release.send(()).unwrap();
+ assert_eq!(mint.await.unwrap().unwrap().status, "refreshed");
+ tokio::time::timeout(Duration::from_secs(10), sweep)
+ .await
+ .unwrap()
+ .unwrap();
+ let committed = get_refresh_state(
+ &store,
+ "default",
+ provider.object_id(),
+ "MS_GRAPH_ACCESS_TOKEN",
+ )
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(committed.status, "refreshed");
+ assert_ne!(committed.next_refresh_at_ms, i64::MAX);
+ assert!(committed.pending_secret_deletions.is_empty());
+ assert_eq!(
+ credentials
+ .resolve_refresh_material(
+ refresh_material_scope(&committed),
+ &committed.secret_material_handles,
+ )
+ .await
+ .unwrap()["refresh_token"],
+ "replacement-grant"
+ );
+ assert_eq!(credentials.stored_credential_count(), Some(2));
+ issuer.verify().await;
+ }
+
#[tokio::test]
async fn rotated_refresh_token_store_failure_requires_reauthorization() {
let mock_server = MockServer::start().await;
@@ -3466,6 +3913,48 @@ mod tests {
);
}
+ #[test]
+ fn automatic_refresh_honors_recovery_schedule_without_disabling_due_retries() {
+ let now_ms = 10_000;
+ for recovery in [
+ ProviderCredentialRefreshRecoveryAction::Retry,
+ ProviderCredentialRefreshRecoveryAction::FixConfiguration,
+ ProviderCredentialRefreshRecoveryAction::Investigate,
+ ] {
+ let mut state = StoredProviderCredentialRefreshState {
+ recovery_action: recovery as i32,
+ next_refresh_at_ms: now_ms + 1,
+ ..Default::default()
+ };
+ assert_eq!(
+ ensure_automatic_refresh_ready(&state, now_ms)
+ .unwrap_err()
+ .code(),
+ tonic::Code::Unavailable,
+ );
+ state.next_refresh_at_ms = now_ms;
+ ensure_automatic_refresh_ready(&state, now_ms).unwrap();
+ }
+ let state = StoredProviderCredentialRefreshState {
+ recovery_action: ProviderCredentialRefreshRecoveryAction::Reauthorize as i32,
+ next_refresh_at_ms: now_ms,
+ ..Default::default()
+ };
+ assert_eq!(
+ ensure_automatic_refresh_ready(&state, now_ms)
+ .unwrap_err()
+ .code(),
+ tonic::Code::FailedPrecondition,
+ );
+ // A healthy credential's proactive schedule must not block an on-read
+ // mint requested to meet a longer minimum remaining lifetime.
+ let state = StoredProviderCredentialRefreshState {
+ next_refresh_at_ms: now_ms + 1,
+ ..Default::default()
+ };
+ ensure_automatic_refresh_ready(&state, now_ms).unwrap();
+ }
+
#[tokio::test]
async fn refresh_worker_skips_parked_reauthorization_state() {
let store = test_store().await;
diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs
index 493fab2b1c..7c96f81787 100644
--- a/crates/openshell-server/src/storage_proto.rs
+++ b/crates/openshell-server/src/storage_proto.rs
@@ -142,8 +142,9 @@ mod tests {
// profile credentials reachable from stored policies and provider profiles.
// Legacy payloads decode empty owners; the gateway rebuilds their authority
// from effective policy rather than trusting persisted owner stamps.
+ // Operator credential export adds four public messages and no durable types.
const PUBLIC_RPC_SCHEMA_SHA256: &str =
- "3fead4a66e57e6828072109564fa25b5b65ef541b35098d9ed7f0785387fe0aa";
+ "f986011b0ced61066dd0787dd86ca50f44c621ce51abc885eef2835160a71109";
const DURABLE_SCHEMA_SHA256: &str =
"96269474903e077df4d4861db0dd1004b8a7205604ffadcdaff98d0124f18147";
const PUBLIC_DURABLE_OVERLAP_SHA256: &str =
@@ -555,12 +556,12 @@ mod tests {
}
assert_eq!(
compiled_method_count,
- 102 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len(),
+ 103 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len(),
"classify every compiled RPC"
);
assert_eq!(
methods.len(),
- 77 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len(),
+ 78 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len(),
"inventory every public gateway RPC"
);
assert_eq!(
@@ -568,7 +569,7 @@ mod tests {
.iter()
.filter(|method| method.starts_with("openshell.v1.OpenShell/"))
.count(),
- 77 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len()
+ 78 + PROVIDER_READINESS_RPC_SIGNATURES.len() + PEER_OWNER_RPC_SIGNATURES.len()
);
assert!(methods.iter().all(|method| !method.contains(".storage.")));
@@ -612,7 +613,7 @@ mod tests {
overlap_hash.as_str(),
),
(
- (307, 27),
+ (311, 27),
(93, 21),
(81, 21),
PUBLIC_RPC_SCHEMA_SHA256,
diff --git a/crates/openshell-server/tests/common/mod.rs b/crates/openshell-server/tests/common/mod.rs
index 414251734d..146169d7ba 100644
--- a/crates/openshell-server/tests/common/mod.rs
+++ b/crates/openshell-server/tests/common/mod.rs
@@ -327,6 +327,13 @@ impl OpenShell for TestOpenShell {
))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
_request: tonic::Request,
diff --git a/crates/openshell-server/tests/supervisor_relay_integration.rs b/crates/openshell-server/tests/supervisor_relay_integration.rs
index de0cfee394..dfb89f18c3 100644
--- a/crates/openshell-server/tests/supervisor_relay_integration.rs
+++ b/crates/openshell-server/tests/supervisor_relay_integration.rs
@@ -361,6 +361,13 @@ impl OpenShell for RelayGateway {
) -> Result, Status> {
Err(Status::unimplemented("unused"))
}
+ async fn get_provider_credentials(
+ &self,
+ _request: tonic::Request,
+ ) -> Result, Status> {
+ Err(Status::unimplemented("not used by this mock"))
+ }
+
async fn get_provider(
&self,
_: tonic::Request,
diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego
index 3389abef37..3c98c7c2e9 100644
--- a/crates/openshell-supervisor-network/data/sandbox-policy.rego
+++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego
@@ -1086,7 +1086,9 @@ endpoint_path_matches_request(ep, request) if {
endpoint_path_matches_request(ep, request) if {
path := object.get(ep, "path", "")
path != ""
- path_matches(request.path, path)
+ # Registered by the embedded engine; use the same Rust matcher as route
+ # selection. Rule paths retain the distinct path_matches semantics above.
+ openshell.endpoint_path_matches(path, request.path)
}
# An endpoint has extended config if it specifies an L7 protocol, allowed_ips,
diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs
index 038f86fd56..31ac623352 100644
--- a/crates/openshell-supervisor-network/src/l7/relay.rs
+++ b/crates/openshell-supervisor-network/src/l7/relay.rs
@@ -8613,6 +8613,136 @@ network_policies:
.unwrap();
}
+ async fn assert_endpoint_path_deny(selector: &str, method: &str, path: &str, denied: bool) {
+ let data = format!(
+ r#"
+network_policies:
+ route_api:
+ endpoints:
+ - host: gateway.example.test
+ port: 443
+ path: "{selector}"
+ protocol: rest
+ enforcement: enforce
+ rules:
+ - allow: {{ method: "*", path: "**" }}
+ deny_rules:
+ - {{ method: POST, path: "**" }}
+ - host: gateway.example.test
+ port: 443
+ protocol: rest
+ enforcement: enforce
+ rules:
+ - allow: {{ method: "*", path: "**" }}
+ binaries:
+ - {{ path: /usr/bin/node }}
+"#,
+ );
+ let engine = OpaEngine::from_strings(TEST_POLICY, &data)
+ .expect("compatible overlapping endpoints must be accepted");
+ let input = NetworkInput {
+ host: "gateway.example.test".into(),
+ port: 443,
+ binary_path: PathBuf::from("/usr/bin/node"),
+ binary_sha256: "unused".into(),
+ ancestors: vec![],
+ cmdline_paths: vec![],
+ };
+ let (endpoints, generation) = engine
+ .query_endpoint_configs_with_generation(&input)
+ .unwrap();
+ let configs: Vec<_> = endpoints
+ .iter()
+ .map(|endpoint| crate::l7::parse_l7_config(endpoint).unwrap())
+ .collect();
+ assert_eq!(configs.len(), 2);
+ if denied {
+ assert_eq!(
+ select_l7_config_for_path(&configs, path).unwrap().path,
+ selector
+ );
+ }
+ let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap();
+ let ctx = L7EvalContext {
+ host: "gateway.example.test".into(),
+ port: 443,
+ request_default_port: Some(443),
+ policy_name: "route_api".into(),
+ binary_path: "/usr/bin/node".into(),
+ ..Default::default()
+ };
+ let (mut app, mut relay_client) = tokio::io::duplex(8192);
+ let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192);
+ let relay = tokio::spawn(async move {
+ relay_with_route_selection(
+ &configs,
+ tunnel_engine,
+ &mut relay_client,
+ &mut relay_upstream,
+ &ctx,
+ )
+ .await
+ });
+ // Respond to any forwarded request so a bypass produces an observable
+ // 200, rather than a timeout that could hide the authorization result.
+ let server = tokio::spawn(async move {
+ let mut request = [0u8; 8192];
+ let n = upstream.read(&mut request).await.unwrap();
+ if n != 0 {
+ upstream
+ .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+ .await
+ .unwrap();
+ }
+ n
+ });
+ app.write_all(
+ format!("{method} {path} HTTP/1.1\r\nHost: gateway.example.test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
+ .as_bytes(),
+ )
+ .await
+ .unwrap();
+ let response = tokio::time::timeout(
+ std::time::Duration::from_secs(5),
+ read_http_headers(&mut app),
+ )
+ .await
+ .expect("relay must answer the request");
+ drop(app);
+ tokio::time::timeout(std::time::Duration::from_secs(5), relay)
+ .await
+ .expect("relay must finish")
+ .unwrap()
+ .unwrap();
+ let forwarded_bytes = server.await.unwrap();
+ let response = String::from_utf8_lossy(&response);
+ let status = if denied { "403 Forbidden" } else { "200 OK" };
+ assert!(
+ response.starts_with(&format!("HTTP/1.1 {status}")),
+ "selector={selector}, request={method} {path}: {response}"
+ );
+ assert_eq!(forwarded_bytes == 0, denied);
+ }
+
+ #[tokio::test]
+ async fn endpoint_path_deny_covers_subtree_root() {
+ assert_endpoint_path_deny("/p/**", "POST", "/p", true).await;
+ }
+
+ #[tokio::test]
+ async fn endpoint_path_deny_covers_multisegment_star() {
+ assert_endpoint_path_deny("/v1/*", "POST", "/v1/a/b", true).await;
+ }
+
+ #[tokio::test]
+ async fn endpoint_path_deny_preserves_matching_and_nonmatching_controls() {
+ assert_endpoint_path_deny("/p/**", "POST", "/p/child", true).await;
+ assert_endpoint_path_deny("/v1/*", "POST", "/v1/a", true).await;
+ assert_endpoint_path_deny("/p/**", "GET", "/p", false).await;
+ assert_endpoint_path_deny("/p/**", "POST", "/public", false).await;
+ assert_endpoint_path_deny("/p/**", "POST", "/prefix", false).await;
+ }
+
/// Policy allowing GET on both `/repos/**` and `/admin/**` for the same
/// host:port, so an encoded-slash denial can only come from the
/// per-endpoint `allow_encoded_slash` scoping.
diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs
index 1779830585..62cf0582e5 100644
--- a/crates/openshell-supervisor-network/src/opa.rs
+++ b/crates/openshell-supervisor-network/src/opa.rs
@@ -6,6 +6,9 @@
//! Wraps [`regorus::Engine`] to evaluate Rego policies for sandbox network
//! access decisions. The engine is loaded once at sandbox startup and queried
//! on every proxy CONNECT request.
+//!
+//! The baked policy uses the `openshell.endpoint_path_matches` native builtin
+//! registered here to share endpoint selector semantics with Rust route selection.
use miette::Result;
use openshell_core::host_pattern::HostSelector;
@@ -36,6 +39,24 @@ mod raw_schema;
/// passthroughs. They reference `data.sandbox.*` for policy data.
const BAKED_POLICY_RULES: &str = include_str!("../data/sandbox-policy.rego");
+fn new_regorus_engine() -> Result {
+ let mut engine = regorus::Engine::new();
+ engine
+ .add_extension(
+ "openshell.endpoint_path_matches".into(),
+ 2,
+ Box::new(|params: Vec| {
+ let pattern = params[0].as_string()?;
+ let path = params[1].as_string()?;
+ Ok(regorus::Value::from(
+ openshell_core::endpoint_path::matches(pattern, path),
+ ))
+ }),
+ )
+ .map_err(|_| miette::miette!("failed to register endpoint path matcher"))?;
+ Ok(engine)
+}
+
/// Maximum number of fixed categories in one policy-load error.
const POLICY_VALIDATION_DIAGNOSTIC_MAX_ITEMS: usize = 8;
/// Maximum byte length of one policy-load error, including its omission marker.
@@ -358,7 +379,7 @@ impl OpaEngine {
// original error, including its source chain, at the load boundary.
let yaml_str = std::fs::read_to_string(data_path)
.map_err(|_| miette::miette!("failed to read YAML policy data file"))?;
- let mut engine = regorus::Engine::new();
+ let mut engine = new_regorus_engine()?;
engine
.add_policy_from_file(policy_path)
.map_err(|_| miette::miette!("failed to load Rego policy"))?;
@@ -405,7 +426,7 @@ impl OpaEngine {
require_binary_identity: bool,
validate_middleware_config: Option<&MiddlewareConfigValidator>,
) -> Result {
- let mut engine = regorus::Engine::new();
+ let mut engine = new_regorus_engine()?;
engine
.add_policy("policy.rego".into(), policy.into())
.map_err(|_| miette::miette!("failed to load Rego policy"))?;
@@ -499,7 +520,7 @@ impl OpaEngine {
emit_l7_config_warnings(&expansion_warnings, "L7 access preset expansion warning");
let data_json = data.to_string();
- let mut engine = regorus::Engine::new();
+ let mut engine = new_regorus_engine()?;
engine
.add_policy("policy.rego".into(), BAKED_POLICY_RULES.into())
.map_err(|_| miette::miette!("failed to load Rego policy"))?;
@@ -5440,7 +5461,7 @@ process:
}
fn eval_l7_raw_data(data: serde_json::Value, input: serde_json::Value) -> bool {
- let mut engine = regorus::Engine::new();
+ let mut engine = new_regorus_engine().unwrap();
engine
.add_policy("policy.rego".into(), TEST_POLICY.into())
.unwrap();
@@ -5457,7 +5478,7 @@ process:
fn token_grant_owner_engine(policies: serde_json::Value) -> OpaEngine {
// Owner identities are trusted runtime metadata, so these Rego tests
// load the already-normalized data that follows policy conversion.
- let mut engine = regorus::Engine::new();
+ let mut engine = new_regorus_engine().expect("register runtime builtins");
engine
.add_policy("policy.rego".into(), TEST_POLICY.into())
.expect("load owner admission policy");
@@ -6024,7 +6045,7 @@ network_policies:
"#;
let data_json: serde_json::Value =
serde_yml::from_str(data).expect("fixture should parse as YAML");
- let mut rego = regorus::Engine::new();
+ let mut rego = new_regorus_engine().unwrap();
rego.add_policy("policy.rego".into(), TEST_POLICY.into())
.expect("policy should load");
rego.add_data_json(&data_json.to_string())
@@ -6097,6 +6118,91 @@ network_policies:
);
}
+ #[test]
+ fn endpoint_path_matching_uses_canonical_rust_semantics() {
+ let engine = OpaEngine::from_strings(TEST_POLICY, "network_policies: {}\n").unwrap();
+ let mut rego = engine.engine.lock().unwrap();
+ for (pattern, path, expected) in [
+ ("", "/p", true),
+ ("**", "/p/a", true),
+ ("/**", "/p/a", true),
+ ("/p", "/p", true),
+ ("/p", "/p/a", false),
+ ("/p/**", "/p", true),
+ ("/p/**", "/p/", true),
+ ("/p/**", "/p/a/b", true),
+ ("/p/**", "/prefix", false),
+ ("/v1/*", "/v1/a", true),
+ ("/v1/*", "/v1/a/b", true),
+ ("/v1/*", "/v1/a.json", true),
+ ("/v1/*", "/v1/a.b/c.d", true),
+ ("/v1/*", "/v2/a", false),
+ ("/v?/items", "/v1/items", true),
+ ("/v[12]/*", "/v2/a/b", true),
+ // A trailing /** treats its prefix literally, including glob syntax.
+ ("/v*/**", "/v1/a", false),
+ ("/v*/**", "/v*/a", true),
+ // Rust glob treats braces literally and invalid patterns as exact paths.
+ ("/a{b,c}", "/ab", false),
+ ("/a{b,c}", "/a{b,c}", true),
+ ("/[", "/[", true),
+ ("/[", "/p", false),
+ ] {
+ assert_eq!(
+ openshell_core::endpoint_path::matches(pattern, path),
+ expected
+ );
+ set_regorus_input(
+ &mut rego,
+ serde_json::json!({
+ "endpoint": { "path": pattern },
+ "request": { "path": path },
+ }),
+ )
+ .unwrap();
+ let result = rego
+ .eval_query(
+ "data.openshell.sandbox.endpoint_path_matches_request(input.endpoint, input.request)".into(),
+ false,
+ )
+ .unwrap();
+ assert_eq!(
+ !result.result.is_empty(),
+ expected,
+ "endpoint selector {pattern:?}, request path {path:?}",
+ );
+ }
+ }
+
+ #[test]
+ fn endpoint_path_matching_preserves_rule_path_globs() {
+ let engine = OpaEngine::from_strings(TEST_POLICY, "network_policies: {}\n").unwrap();
+ let mut rego = engine.engine.lock().unwrap();
+ for (pattern, path, expected) in [
+ ("/p/**", "/p", false),
+ ("/p/**", "/p/a/b", true),
+ ("/v1/*", "/v1/a", true),
+ ("/v1/*", "/v1/a/b", false),
+ ] {
+ set_regorus_input(
+ &mut rego,
+ serde_json::json!({ "pattern": pattern, "path": path }),
+ )
+ .unwrap();
+ let result = rego
+ .eval_query(
+ "data.openshell.sandbox.path_matches(input.path, input.pattern)".into(),
+ false,
+ )
+ .unwrap();
+ assert_eq!(
+ !result.result.is_empty(),
+ expected,
+ "rule pattern {pattern:?}, request path {path:?}",
+ );
+ }
+ }
+
#[test]
fn l7_method_matching_case_insensitive() {
let engine = l7_engine();
diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs
index 56c9809cbc..ac59b7e3c5 100644
--- a/crates/openshell-supervisor-network/src/policy_local.rs
+++ b/crates/openshell-supervisor-network/src/policy_local.rs
@@ -689,30 +689,44 @@ fn sanitize_reason_for_audit(raw: &str) -> String {
}
/// One-line audit description of a chunk's target: binary, host, port, and
-/// L7 method/path if present. Used by both the propose and approve/reject
-/// audit events so the trace can be grepped by endpoint without parsing
-/// JSON.
+/// L7 method/path allows for every endpoint. Used by both the propose and
+/// approve/reject audit events so the trace can be grepped by endpoint
+/// without parsing JSON.
fn summarize_chunk_for_audit(chunk: &PolicyChunk) -> String {
let Some(rule) = chunk.proposed_rule.as_ref() else {
return format!("rule_name:{}", chunk.rule_name);
};
- let endpoint = rule.endpoints.first().map_or_else(
- || "unknown".to_string(),
- |ep| format!("{}:{}", ep.host, ep.port),
- );
- let l7 = rule
- .endpoints
- .first()
- .and_then(|ep| ep.rules.first())
- .and_then(|r| r.allow.as_ref())
- .map(|a| format!(" {} {}", a.method, a.path))
- .unwrap_or_default();
+ let endpoints = if rule.endpoints.is_empty() {
+ "unknown".to_string()
+ } else {
+ rule.endpoints
+ .iter()
+ .map(summarize_endpoint_for_audit)
+ .collect::>()
+ .join(", ")
+ };
let binary = if chunk.binary.is_empty() {
String::new()
} else {
format!(" by {}", chunk.binary)
};
- format!("on {endpoint}{l7}{binary}")
+ format!("on {endpoints}{binary}")
+}
+
+fn summarize_endpoint_for_audit(endpoint: &NetworkEndpoint) -> String {
+ let allows = endpoint
+ .rules
+ .iter()
+ .filter_map(|r| r.allow.as_ref())
+ .map(|a| format!("{} {}", a.method, a.path))
+ .collect::>()
+ .join(", ");
+ let target = format!("{}:{}", endpoint.host, endpoint.port);
+ if allows.is_empty() {
+ target
+ } else {
+ format!("{target} [{allows}]")
+ }
}
/// `GET /v1/proposals/{chunk_id}` — immediate state. One gateway call, no loop.
@@ -2053,6 +2067,42 @@ mod tests {
assert!(summary.contains("/usr/bin/curl"));
}
+ #[test]
+ fn summarize_chunk_for_audit_lists_every_endpoint_and_allow() {
+ let allow = |method: &str, path: &str| L7Rule {
+ allow: Some(L7Allow {
+ method: method.to_string(),
+ path: path.to_string(),
+ ..Default::default()
+ }),
+ };
+ let chunk = PolicyChunk {
+ binary: "/usr/bin/perl".to_string(),
+ proposed_rule: Some(NetworkPolicyRule {
+ name: "two".to_string(),
+ endpoints: vec![
+ NetworkEndpoint {
+ host: "ports.ubuntu.com".to_string(),
+ port: 80,
+ rules: vec![allow("GET", "/ubuntu-ports/**"), allow("HEAD", "/x")],
+ ..Default::default()
+ },
+ NetworkEndpoint {
+ host: "169.254.169.254".to_string(),
+ port: 80,
+ ..Default::default()
+ },
+ ],
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ assert_eq!(
+ summarize_chunk_for_audit(&chunk),
+ "on ports.ubuntu.com:80 [GET /ubuntu-ports/**, HEAD /x], 169.254.169.254:80 by /usr/bin/perl"
+ );
+ }
+
// Helpers — synthetic proposed rule + policy with that rule already
// merged. Both reused across reload-readiness tests.
fn proposed_curl_rule_for_github() -> NetworkPolicyRule {
diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs
index db6ac2afe3..75c7b1fa5c 100644
--- a/crates/openshell-supervisor-process/src/ssh.rs
+++ b/crates/openshell-supervisor-process/src/ssh.rs
@@ -652,14 +652,19 @@ impl russh::server::Handler for SshHandler {
channel: ChannelId,
session: &mut Session,
) -> Result<(), Self::Error> {
- session.channel_success(channel)?;
// Only allocate a PTY when the client explicitly requested one via
// pty_request. VS Code Remote-SSH sends shell_request *without* a
// preceding pty_request and expects pipe-based I/O with clean LF line
// endings. Forcing a PTY here caused CRLF translation which made
// VS Code misdetect the platform as Windows (and then try to run
// `powershell`).
- self.start_shell(channel, session.handle(), None).await?;
+ match self.start_shell(channel, session.handle(), None).await {
+ Ok(()) => session.channel_success(channel)?,
+ Err(error) => {
+ warn!(%error, ?channel, "failed to start boundary shell");
+ session.channel_failure(channel)?;
+ }
+ }
Ok(())
}
@@ -669,13 +674,21 @@ impl russh::server::Handler for SshHandler {
data: &[u8],
session: &mut Session,
) -> Result<(), Self::Error> {
- session.channel_success(channel)?;
let command = String::from_utf8_lossy(data).trim().to_string();
if command.is_empty() {
+ session.channel_success(channel)?;
return Ok(());
}
- self.start_shell(channel, session.handle(), Some(command))
- .await?;
+ match self
+ .start_shell(channel, session.handle(), Some(command))
+ .await
+ {
+ Ok(()) => session.channel_success(channel)?,
+ Err(error) => {
+ warn!(%error, ?channel, "failed to start boundary exec");
+ session.channel_failure(channel)?;
+ }
+ }
Ok(())
}
diff --git a/crates/openshell-supervisor-process/src/ssh/exec_input_tests.rs b/crates/openshell-supervisor-process/src/ssh/exec_input_tests.rs
index 4aea1c4cde..67e04dedfa 100644
--- a/crates/openshell-supervisor-process/src/ssh/exec_input_tests.rs
+++ b/crates/openshell-supervisor-process/src/ssh/exec_input_tests.rs
@@ -12,6 +12,7 @@ use openshell_isolation_interface::contract::{
use russh::ChannelMsg;
use std::collections::VecDeque;
use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use std::time::Duration;
@@ -96,6 +97,28 @@ impl BoundaryExec for TestExec {
}
}
+struct FailThenExec {
+ failed: AtomicBool,
+ session: Mutex