diff --git a/.config/mise/tasks/artifacts/wasm-components b/.config/mise/tasks/artifacts/wasm-components new file mode 100755 index 000000000..b4f1d98d7 --- /dev/null +++ b/.config/mise/tasks/artifacts/wasm-components @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#MISE description="Build every rsworkspace/wasm-components/* decider to dist/decider/.wasm" +#MISE depends=["proto:schedules-descriptor-set", "rust:wasm-target"] +# +# `proto:schedules-descriptor-set` only feeds the schedules component's own dist +# artifact (copied below when present); it is not required by, and does not +# block, other components. +set -euo pipefail + +ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +cd "$ROOT/rsworkspace" + +mkdir -p dist/decider +for manifest in wasm-components/*/Cargo.toml; do + crate_dir="$(dirname "$manifest")" + crate="$(basename "$crate_dir")" + crate_file="${crate//-/_}" + alias="${crate#trogon-}" + alias="${alias%-decider}" + cargo component build -p "$crate" --target wasm32-unknown-unknown --release + cp "target/wasm32-unknown-unknown/release/${crate_file}.wasm" "dist/decider/${alias}.wasm" + binpb="target/wasm32-unknown-unknown/release/${crate_file}.binpb" + if [[ -f "$binpb" ]]; then + cp "$binpb" "dist/decider/${alias}.binpb" + fi +done diff --git a/.config/mise/tasks/github-actions/build-release-binary b/.config/mise/tasks/github-actions/build-release-binary new file mode 100755 index 000000000..ecea5a029 --- /dev/null +++ b/.config/mise/tasks/github-actions/build-release-binary @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +#MISE description="Build the acp-nats-stdio release binary for TARGET, via cross when USE_CROSS=true" +set -euo pipefail + +: "${USE_CROSS:?USE_CROSS is required}" +: "${TARGET:?TARGET is required}" + +ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +cd "$ROOT/rsworkspace" + +if [[ "$USE_CROSS" == "true" ]]; then + cross build --release --target "$TARGET" -p acp-nats-stdio +else + cargo build --release --target "$TARGET" -p acp-nats-stdio +fi diff --git a/.config/mise/tasks/github-actions/decider-test-suites b/.config/mise/tasks/github-actions/decider-test-suites new file mode 100755 index 000000000..9a6ef4115 --- /dev/null +++ b/.config/mise/tasks/github-actions/decider-test-suites @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +#MISE description="Run wasm decider conformance: sim schedules suite, macro compile-fail tests, and every decider-test YAML fixture" +set -euo pipefail + +ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +cd "$ROOT/rsworkspace" + +cargo test -p trogon-decider-sim --test schedules --features test-support +cargo test -p trogon-decider-guest-macros --test compile_fail +cargo build -p trogon-decider-test --release + +shopt -s nullglob +for wasm in dist/decider/*.wasm; do + alias="$(basename "$wasm" .wasm)" + yaml="cli/trogon-decider-test/${alias}.yaml" + if [[ -f "$yaml" ]]; then + ./target/release/decider-test "$wasm" "$yaml" + else + echo "no decider-test fixture for $alias ($yaml not found); skipping" + fi +done diff --git a/.config/mise/tasks/github-actions/dylint-tests b/.config/mise/tasks/github-actions/dylint-tests new file mode 100755 index 000000000..2ecdea433 --- /dev/null +++ b/.config/mise/tasks/github-actions/dylint-tests @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +#MISE description="Run the repo Rust policy lint crate's own tests on its pinned dylint toolchain" +set -euo pipefail + +ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +cd "$ROOT/rsworkspace/dylints/trogon_lints" + +dylint_toolchain="$(python - <<'PY' +import tomllib + +with open("rust-toolchain.toml", "rb") as toolchain_file: + print(tomllib.load(toolchain_file)["toolchain"]["channel"]) +PY +)" + +rustup run "$dylint_toolchain" cargo test diff --git a/.config/mise/tasks/github-actions/image-audit-summary b/.config/mise/tasks/github-actions/image-audit-summary new file mode 100755 index 000000000..efd4e2e8f --- /dev/null +++ b/.config/mise/tasks/github-actions/image-audit-summary @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +#MISE description="Append the canary image security audit report to GITHUB_STEP_SUMMARY" +set -euo pipefail + +: "${SERVICE:?SERVICE is required}" +: "${IMAGE_NAME:?IMAGE_NAME is required}" +: "${IMAGE_DIGEST:?IMAGE_DIGEST is required}" +: "${RUNTIME_BASE:?RUNTIME_BASE is required}" +: "${GITHUB_STEP_SUMMARY:?GITHUB_STEP_SUMMARY is required}" + +{ + echo "## Image security audit: \`${SERVICE}\`" + echo "" + echo "- Image: \`${IMAGE_NAME}@${IMAGE_DIGEST}\`" + echo "- Runtime base: \`${RUNTIME_BASE}\`" + echo "- SBOM artifact: \`sbom-${SERVICE}-spdx\` (SPDX JSON)" + echo "" + echo "### Vulnerability scan (Trivy, image packages)" + echo '```text' + cat trivy-results.txt + echo '```' +} >>"$GITHUB_STEP_SUMMARY" diff --git a/.config/mise/tasks/github-actions/package-release-binary b/.config/mise/tasks/github-actions/package-release-binary new file mode 100755 index 000000000..9d422aad2 --- /dev/null +++ b/.config/mise/tasks/github-actions/package-release-binary @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +#MISE description="Package the built acp-nats-stdio binary into a versioned tarball and export ASSET via GITHUB_ENV" +set -euo pipefail + +: "${VERSION:?VERSION is required}" +: "${TARGET:?TARGET is required}" +: "${GITHUB_ENV:?GITHUB_ENV is required}" + +ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +cd "$ROOT" + +staging="acp-nats-stdio-${VERSION}-${TARGET}" +mkdir -p "$staging" +cp "rsworkspace/target/${TARGET}/release/acp-nats-stdio" "$staging/" +tar czf "$staging.tar.gz" "$staging" +echo "ASSET=$staging.tar.gz" >>"$GITHUB_ENV" diff --git a/.config/mise/tasks/github-actions/release-upload b/.config/mise/tasks/github-actions/release-upload new file mode 100755 index 000000000..b1705b07d --- /dev/null +++ b/.config/mise/tasks/github-actions/release-upload @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +#MISE description="Create the GitHub release for the current tag if missing and upload all artifacts" +set -euo pipefail + +: "${GITHUB_REF_NAME:?GITHUB_REF_NAME is required}" +: "${GH_TOKEN:?GH_TOKEN is required}" + +gh release view "$GITHUB_REF_NAME" || + gh release create "$GITHUB_REF_NAME" --title "$GITHUB_REF_NAME" --generate-notes +gh release upload "$GITHUB_REF_NAME" artifacts/* --clobber diff --git a/.config/mise/tasks/github-actions/resolve-image b/.config/mise/tasks/github-actions/resolve-image new file mode 100755 index 000000000..c645f1332 --- /dev/null +++ b/.config/mise/tasks/github-actions/resolve-image @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +#MISE description="Resolve the canary container image name and short SHA into GITHUB_OUTPUT" +set -euo pipefail + +: "${IMAGE_OWNER:?IMAGE_OWNER is required}" +: "${SERVICE_IMAGE:?SERVICE_IMAGE is required}" +: "${GITHUB_SHA:?GITHUB_SHA is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +owner_lower="$(tr '[:upper:]' '[:lower:]' <<<"$IMAGE_OWNER")" +echo "name=ghcr.io/${owner_lower}/${SERVICE_IMAGE}" >>"$GITHUB_OUTPUT" +echo "short_sha=${GITHUB_SHA:0:7}" >>"$GITHUB_OUTPUT" diff --git a/.config/mise/tasks/github-actions/wasm-validate b/.config/mise/tasks/github-actions/wasm-validate new file mode 100755 index 000000000..bf0b7aa86 --- /dev/null +++ b/.config/mise/tasks/github-actions/wasm-validate @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +#MISE description="Assert every built decider component declares zero imports and passes wasm validation" +set -euo pipefail + +ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}" +cd "$ROOT" + +shopt -s nullglob +components=(rsworkspace/dist/decider/*.wasm) +if [[ ${#components[@]} -eq 0 ]]; then + echo "no decider components found under rsworkspace/dist/decider; run 'mise run artifacts:wasm-components' first" >&2 + exit 1 +fi + +for wasm in "${components[@]}"; do + wit_text="$(wasm-tools component wit "$wasm")" + printf '%s\n' "$wit_text" + if grep -E '^[[:space:]]*import ' <<<"$wit_text"; then + echo "component declares imports: $wasm" >&2 + exit 1 + fi + wasm-tools validate "$wasm" +done diff --git a/.github/workflows/canary-container-images.yml b/.github/workflows/canary-container-images.yml index 1c6dd783d..b6649aaa5 100644 --- a/.github/workflows/canary-container-images.yml +++ b/.github/workflows/canary-container-images.yml @@ -40,9 +40,8 @@ jobs: id: image env: IMAGE_OWNER: ${{ github.repository_owner }} - run: | - echo "name=ghcr.io/${IMAGE_OWNER,,}/${{ steps.service_config.outputs.image }}" >> "$GITHUB_OUTPUT" - echo "short_sha=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + SERVICE_IMAGE: ${{ steps.service_config.outputs.image }} + run: ./.config/mise/tasks/github-actions/resolve-image - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 @@ -88,16 +87,9 @@ jobs: - name: Document image security audit shell: bash - run: | - { - echo "## Image security audit: \`${{ matrix.service }}\`" - echo "" - echo "- Image: \`${{ steps.image.outputs.name }}@${{ steps.build_image.outputs.digest }}\`" - echo "- Runtime base: \`debian:bookworm-20260518-slim\`" - echo "- SBOM artifact: \`sbom-${{ matrix.service }}-spdx\` (SPDX JSON)" - echo "" - echo "### Vulnerability scan (Trivy, image packages)" - echo '```text' - cat trivy-results.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + env: + SERVICE: ${{ matrix.service }} + IMAGE_NAME: ${{ steps.image.outputs.name }} + IMAGE_DIGEST: ${{ steps.build_image.outputs.digest }} + RUNTIME_BASE: debian:bookworm-20260518-slim + run: ./.config/mise/tasks/github-actions/image-audit-summary diff --git a/.github/workflows/ci-rust.yml b/.github/workflows/ci-rust.yml index bc45734cd..5307096b1 100644 --- a/.github/workflows/ci-rust.yml +++ b/.github/workflows/ci-rust.yml @@ -65,16 +65,7 @@ jobs: working-directory: rsworkspace - name: Test repo Rust policy lints - run: | - dylint_toolchain="$(python - <<'PY' - import tomllib - - with open("rust-toolchain.toml", "rb") as toolchain_file: - print(tomllib.load(toolchain_file)["toolchain"]["channel"]) - PY - )" - rustup run "$dylint_toolchain" cargo test - working-directory: rsworkspace/dylints/trogon_lints + run: mise run github-actions:dylint-tests - name: Run repo Rust policy lints run: env -u RUSTUP_TOOLCHAIN cargo dylint --path dylints/trogon_lints --workspace --no-deps -- --all-features --all-targets @@ -115,8 +106,7 @@ jobs: tool: cargo-llvm-cov,nextest - name: Build decider components for sim fixtures - run: cargo component build -p trogon-schedules-decider --target wasm32-unknown-unknown --release - working-directory: rsworkspace + run: mise run artifacts:wasm-components - name: Run documentation tests run: cargo test --doc @@ -187,28 +177,14 @@ jobs: workspaces: rsworkspace -> target save-if: always - - name: Build schedules decider component - run: cargo component build -p trogon-schedules-decider --target wasm32-unknown-unknown --release - working-directory: rsworkspace + - name: Build decider components + run: mise run artifacts:wasm-components - name: Assert zero imports and validate - run: | - for WASM in rsworkspace/target/wasm32-unknown-unknown/release/trogon_schedules_decider.wasm; do - wasm-tools component wit "$WASM" | tee /tmp/wit.txt - if grep -E '^[[:space:]]*import ' /tmp/wit.txt; then - echo "component declares imports: $WASM" - exit 1 - fi - wasm-tools validate "$WASM" - done + run: mise run github-actions:wasm-validate - name: Run decider-test suites - run: | - cd rsworkspace - cargo test -p trogon-decider-sim --test schedules --features test-support - cargo test -p trogon-decider-guest-macros --test compile_fail - cargo build -p trogon-decider-test --release - ./target/release/decider-test target/wasm32-unknown-unknown/release/trogon_schedules_decider.wasm cli/trogon-decider-test/schedules.yaml + run: mise run github-actions:decider-test-suites - name: Run wasm decider runtime tests run: cargo test -p trogon-decider-wasm-runtime diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index 1e9b8b9d3..d6977bd82 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -59,29 +59,18 @@ jobs: tool: cross - name: Build - working-directory: rsworkspace shell: bash env: USE_CROSS: ${{ matrix.use-cross }} TARGET: ${{ matrix.target }} - run: | - if [ "$USE_CROSS" = "true" ]; then - cross build --release --target "$TARGET" -p acp-nats-stdio - else - cargo build --release --target "$TARGET" -p acp-nats-stdio - fi + run: mise run github-actions:build-release-binary - name: Package shell: bash env: VERSION: ${{ steps.version.outputs.version }} TARGET: ${{ matrix.target }} - run: | - staging="acp-nats-stdio-${VERSION}-${TARGET}" - mkdir -p "$staging" - cp "rsworkspace/target/${TARGET}/release/acp-nats-stdio" "$staging/" - tar czf "$staging.tar.gz" "$staging" - echo "ASSET=$staging.tar.gz" >> "$GITHUB_ENV" + run: mise run github-actions:package-release-binary - name: Attest build provenance uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 @@ -119,7 +108,4 @@ jobs: - name: Upload to GitHub Release env: GH_TOKEN: ${{ github.token }} - run: | - gh release view "$GITHUB_REF_NAME" || \ - gh release create "$GITHUB_REF_NAME" --title "$GITHUB_REF_NAME" --generate-notes - gh release upload "$GITHUB_REF_NAME" artifacts/* --clobber + run: ./.config/mise/tasks/github-actions/release-upload diff --git a/.mise.toml b/.mise.toml index 6d90952e5..75694b83d 100644 --- a/.mise.toml +++ b/.mise.toml @@ -69,14 +69,9 @@ description = "Enforce ADR#0015: fail if native-tls/OpenSSL enter the Rust depen dir = "rsworkspace" run = "cargo deny check bans" -[tasks."artifacts:schedules-wasm"] -description = "Build schedules decider WASM artifact to dist/decider/" -depends = ["proto:schedules-descriptor-set", "rust:wasm-target"] -run = "cd rsworkspace && cargo component build -p trogon-schedules-decider --target wasm32-unknown-unknown --release && mkdir -p dist/decider && cp target/wasm32-unknown-unknown/release/trogon_schedules_decider.wasm dist/decider/schedules.wasm && cp target/wasm32-unknown-unknown/release/trogon_schedules_decider.binpb dist/decider/schedules.binpb" - [tasks."artifacts:decider-test"] -description = "Build schedules decider WASM + decider-test CLI" -depends = ["artifacts:schedules-wasm"] +description = "Build every wasm-components/* decider + the decider-test CLI" +depends = ["artifacts:wasm-components"] run = "cd rsworkspace && cargo build -p trogon-decider-test --release" [tasks."rsc"] diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 38f351b72..10748797a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -35,6 +35,7 @@ export default async () => { text: "Architecture", items: [ { text: "ACP Conformance", link: "/architecture/acp-conformance" }, + { text: "Decider", link: "/architecture/decider" }, { text: "Event Metadata", link: "/architecture/event-metadata" }, ], }, diff --git a/docs/adr/0026-command-authorization-principal.md b/docs/adr/0026-command-authorization-principal.md new file mode 100644 index 000000000..a74124f4a --- /dev/null +++ b/docs/adr/0026-command-authorization-principal.md @@ -0,0 +1,178 @@ +--- +number: "0026" +slug: command-authorization-principal +status: draft +date: 2026-07-15 +--- + +# ADR#0026: Command Authorization Principal and Authorizer Hook for Decider Execution + +## Context + +`trogon_decider_runtime::execution::CommandExecution` is the single entry +point every native command execution goes through: it loads history, replays +it, calls `Decider::decide`, encodes the resulting events, and appends them. +Its builder carries an event store, the command, an optional write +precondition, an optional snapshot policy, an event id generator, and +`headers: Headers` -- a freeform, caller-supplied metadata bag. Nothing in +that builder, nothing in the `Decider` trait itself, and nothing in the WASM +mirror (`trogon_decider_wasm_runtime::execution::WasmCommandExecution`, which +has the identical shape) represents who is submitting a command or what they +are allowed to do. `decide` runs against decider state and the command +payload only. + +`docs/architecture/event-metadata.md` is explicit that the runtime should not +derive headers generically: an application that wants a fixed header +(tenancy, correlation, and by the same reasoning, caller identity) must build +it itself before calling `CommandExecution::with_headers`. In practice this +means the closest thing to "who acted" today is whatever string an +application chooses to put in a header, which `decide` never sees and which +carries no authorization semantics -- it is envelope metadata for storage, not +an input the runtime checks before evaluating a decision. + +Command execution is also reached from more than one direction. The A2A +gateway resolves caller identity at ingress +([ADR#0017](./0017-aauth-agent-authentication.md)), but internal callers such +as `trogon-scheduler`'s worker processor construct `CommandExecution::new` +directly, off the gateway path entirely. Any authorization hook that only +lives at the gateway leaves every non-gateway caller of `CommandExecution` +unenforced. + +AAuth itself is not a ready-made carrier for this. Its three-party auth token +(`aa-auth+jwt`) does carry a `principal` claim, but on the wire it is an +`Option<&str>` (`trogon-aauth-person::mint::MintInputs::principal`, +mirrored in `trogon-aauth-as`) -- an optional, unstructured string naming a +directed user, with no claim set describing scopes, roles, or a stable +principal kind. That is sufficient for what the draft needs at the protocol +level; it is not something `decide` can safely branch on directly, and it +gives no way to distinguish an agent principal from a person principal or an +absent claim from an empty one. + +## Decision + +### 1. A typed `CommandPrincipal`, separate from `Headers` + +Introduce a `CommandPrincipal` value type carried on `CommandExecution` and +`WasmCommandExecution` through a new builder method +(`with_principal`), distinct from `Headers`. `Headers` stays what +`event-metadata.md` already defines it as: envelope metadata for the stored +event. `CommandPrincipal` is an authorization-time input evaluated before +`decide` runs; it is not required to be persisted verbatim, and applications +that want an audit trail of who acted still derive and set their own header +from it, the same way they derive any other required header today. +`CommandPrincipal` models a principal kind (agent, person, service), +a stable principal identifier, and an opaque claims/scope set -- structured +enough for an authorizer to make a decision, without prescribing a policy +language. + +### 2. An authorizer trait hook that runs before `decide` + +Add a `CommandAuthorizer` trait with one method that takes the +principal and the command and returns either `Ok(())` or a typed denial. +`CommandExecution`/`WasmCommandExecution` gain an optional builder slot for +an authorizer, defaulting to an `AllowAll` no-op so existing callers keep +compiling and behaving exactly as they do today -- the same opt-in shape +`WithoutSnapshots` already uses for the snapshot builder slot. When an +authorizer is configured, `execute()` calls it after the command's stream id +and the loaded state are available (an authorizer may need to know the +target stream) but strictly before `evaluate_decision`/`decide` runs, so a +denied command never reaches domain logic and never spends replay or, on the +WASM path, guest fuel on work that will be rejected. + +### 3. Both native and WASM dispatch paths + +For the WASM path, the authorizer check sits in +`WasmCommandExecution::execute` right after `call_stream_id` resolves the +target stream, before `create_session`/`replay_events`/`decide` run -- +mirroring the native placement and avoiding the cost of instantiating guest +session state for a command that will be denied. The trait itself does not +differ between paths; only where each path can cheaply evaluate it differs, +because the WASM path does not have `state` available in host-native form at +that point (only the stream id and the command envelope). + +### 4. Composing with AAuth given its optional-string principal + +The gateway (or any other ingress boundary that already runs AAuth +verification) is where a verified identity becomes a `CommandPrincipal`: the +PoP-verified agent's `sub`/`cnf.jwk` thumbprint maps to an agent principal, +and an `aa-auth+jwt`'s `principal` string, when present, is carried as an +opaque hint attached to that principal rather than trusted as a scoped claim +on its own. This ADR does not change AAuth's wire shape or fix the +optional-string limitation -- that is ADR#0017's pinned draft shape. The +mapping boundary is the one place that has to absorb the limitation, and it +must fail closed: a missing or unparsable principal where one is required is +a denial, never a silent anonymous principal. + +### 5. Missing or invalid principal + +With no authorizer configured, behavior is unchanged (today's implicit +"anyone can submit any command"). Once an authorizer is configured, a command +executed with no principal, or a principal the authorizer cannot validate, is +a hard failure -- a new `Unauthorized` variant on `CommandError`/ +`WasmCommandError` -- not a fallback to an anonymous or default-trust +principal. + +## Alternatives Considered + +### Encode the principal as another `Headers` entry + +Rejected. `Headers` is a caller-suppliable, un-typed string bag that already +persists into the event envelope; nothing distinguishes a header a verified +ingress layer set from one a client typed by hand, so an authorizer reading +it could not tell a validated identity from a spoofed one. It also conflates +an authorization-time input with the audit-trail output `event-metadata.md` +already defines headers to be. + +### Authorize only at the gateway, trust everything past it + +Rejected. Not every command execution path goes through the A2A gateway -- +`trogon-scheduler`'s worker processor calls `CommandExecution::new` directly. +An authorization hook that only exists at one ingress point leaves every +other caller of the decider runtime unenforced, and the decider crate itself +would have no way to reason about whether a given execution was ever checked. + +### Put the principal on every `Decider`-implementing command struct + +Rejected. `Decider::decide` takes `&self` as the command and already defines +`stream_id()`; adding a principal field to every command type conflates +domain payload (business intent) with cross-cutting authorization context, +forces every decider author to remember to carry and validate it, and cannot +be composed, tested, or swapped independently of the domain type. + +## Non-Goals + +- Defining a policy language (SpiceDB, CEL, Rego, or otherwise). This ADR + defines the extension point a policy engine plugs into, not the engine. +- Replacing AAuth or changing its wire tokens; the `principal` claim's + optional-string shape is AAuth's own pinned draft shape and out of scope + here. +- Authorizing stream reads or snapshot reads/writes independent of command + execution. Only the `decide` entry point is gated. +- Specifying how an application composes multiple authorizers (allow-list, + policy callout, or otherwise). One trait, one hook per execution. + +## Consequences + +- `CommandExecution` and `WasmCommandExecution` gain a new builder slot and, + once populated, a new phase in the execution pipeline; the default no-op + authorizer keeps every existing call site compiling and behaving unchanged. +- `CommandError`/`WasmCommandError` gain an `Unauthorized` variant, additive + but breaking for exhaustive matches on those enums, consistent with how + every other execution phase already gets its own variant. +- Enforcement is opt-in per call site. This ADR does not retroactively close + the "anyone can submit any command" gap the audit identified; it closes it + only where a caller adopts an authorizer. +- Authorization becomes a distinguishable phase in logging and metrics, + separate from a domain rejection (`Decide`) or an infrastructure failure + (`Append`), matching this crate's existing philosophy of phase-tagged + errors. +- Gets harder: the authorizer hook runs on every command execution, including + hot paths, so a slow or blocking authorizer implementation directly adds + latency to every command; this ADR does not mandate a cost bound on + implementations. + +## References + +- [ADR#0017: AAuth Agent Authentication over a Trogon NATS PoP Binding](./0017-aauth-agent-authentication.md) +- [ADR#0023: Secret Management and Key Custody on OpenBao behind a Platform Secrets Service](./0023-secret-management-and-key-custody-direction.md) +- [Event Metadata](../architecture/event-metadata.md) diff --git a/docs/adr/0027-decider-multi-tenancy-primitive.md b/docs/adr/0027-decider-multi-tenancy-primitive.md new file mode 100644 index 000000000..f7fb7f673 --- /dev/null +++ b/docs/adr/0027-decider-multi-tenancy-primitive.md @@ -0,0 +1,147 @@ +--- +number: "0027" +slug: decider-multi-tenancy-primitive +status: draft +date: 2026-07-15 +--- + +# ADR#0027: Tenant Value Object for Decider Stream and Snapshot Resolution + +## Context + +`trogon_decider_nats::JetStreamStore` binds to exactly one physical +JetStream `events_stream` and one physical KV `snapshot_bucket` per store +instance. Isolation across tenants depends entirely on the +`StreamSubjectResolver` implementation an application supplies -- +the trait's own doc comment says implementations "usually compose a tenant or +aggregate prefix with the caller's stream id." "Usually" is the whole +enforcement mechanism. `resolve_subject_state` returns a +`SubjectState { subject, current_position }` with no tenant field anywhere in +it, and `ReadSnapshotRequest`/`WriteSnapshotRequest` carry a bare +`snapshot_id: &StreamId` with the same absence. Nothing in `JetStreamStore` +checks that a resolved subject or snapshot key is actually scoped to the +tenant the caller believes it is operating as; the store reads and writes +whatever the resolver computed. + +The event-metadata story is the same convention, not a type: if a workflow +needs tenancy recorded, `docs/architecture/event-metadata.md` says the +application builds it as a header at the boundary, validated there, before +calling `CommandExecution::with_headers`. The stream store's own tests +demonstrate exactly this pattern with `Headers::from_entries([("tenant", +"trogon"), ...])` -- a plain string key an application chose to set, invisible +to the store, and unrelated to which physical subject or KV key the event +actually landed on. + +A resolver bug, two tenants' resolvers computing overlapping subjects, or a +caller passing an already-prefixed id where an unprefixed one was expected +would silently cross tenant boundaries today, because "tenant" is not a +concept the store or the resolver contract has any way to check against. + +## Decision + +### 1. A `Tenant` value object + +Introduce `Tenant` as a validated, non-empty identifier type in +`trogon_decider_runtime`, alongside the crate's existing value objects +(`StreamPosition`, `EventId`). A `Tenant` is constructed once, at the +application boundary that knows which tenant is operating, the same place +that already assembles `Headers` and a `CommandPrincipal` +([ADR#0026](./0026-command-authorization-principal.md)) before execution. + +### 2. Tenant-aware resolution, verified at the store boundary + +Extend `StreamSubjectResolver` and the snapshot key resolution surface +(currently the bare `&str` snapshot id passed into +`crate::snapshot_store::{read_snapshot, write_snapshot}`) to take a `Tenant` +explicitly, and to return a tenant-scoped result type rather than a bare +`StreamSubject`/`String` -- so "did the resolver honor the tenant" is a +value the store can check, not a hope about the resolver's implementation. +`JetStreamStore` validates the resolved subject/key against the requested +`Tenant` before reading or writing and returns a typed error on mismatch, +turning a resolver bug into a caught error instead of a silent cross-tenant +read or write. + +### 3. Optional per-tenant stream/bucket binding + +`JetStreamStoreBuilder` gains an optional `TenantBinding` (`Shared` or +`Dedicated(events_stream, snapshot_bucket)`), resolved per tenant. Most +tenants keep today's default: one shared physical stream and bucket, +distinguished by subject/key scoping alone. A tenant that needs hard physical +isolation (noisy-neighbor containment, a compliance requirement, or its own +retention posture) can be bound to its own stream and bucket without changing +`JetStreamStore`'s public shape or any caller's resolver code. + +### 4. Migration for existing single-tenant deployments + +An existing deployment supplies one static `Tenant` value at the point where +it constructs its resolver today -- a one-line change, not a data migration. +Its existing subjects and snapshot keys already satisfy the new validation +trivially, because there is exactly one tenant and the resolver's output +already matches it. No historical subject or key is rewritten; validation +applies going forward from adoption, the same posture +[ADR#0013](./0013-origin-stream-sequence-header.md) takes toward provenance +metadata on pre-existing events. + +## Alternatives Considered + +### Keep tenancy as a header-only convention, add lint or test discipline + +Rejected. This is the exact "no design, just a doc comment" state the audit +identified. A header is invisible to the store, which is the actual +isolation boundary for JetStream and KV reads and writes; a lint on +application code cannot catch a resolver bug that computes an unscoped +subject internally. + +### Require a dedicated physical stream and bucket per tenant, always + +Rejected as the default. It forecloses small or low-volume tenants sharing +infrastructure cheaply, which is the reason the current single-shared-stream +design exists, and it is a heavier operational default than most deployments +need. Kept as an opt-in `TenantBinding::Dedicated` rather than the only +supported mode. + +### Enforce tenancy only through NATS account/subject permissions + +Rejected as the sole mechanism. Server-side ACLs protect against a +compromised or malicious client but do nothing for a resolver bug inside a +trusted process holding broad JetStream credentials -- which is this crate's +own client shape. A host-side check catches programming errors ACLs cannot +see, the same reasoning [ADR#0023](./0023-secret-management-and-key-custody-direction.md) +applies to keeping OpenBao access behind a single trusted client rather than +relying on network policy alone. + +## Non-Goals + +- Designing per-tenant NATS accounts, credentials, or ACL policy. That is an + operational/deployment concern layered on top of this primitive. +- Retrofitting historical subjects or snapshot keys written before adoption. + Validation applies going forward only. +- Cross-tenant data sharing or tenant-to-tenant migration tooling. +- Changing the command-authorization boundary + ([ADR#0026](./0026-command-authorization-principal.md)). `Tenant` is a + storage-resolution concept; associating a caller's authorization principal + with a tenant is left to the application boundary. + +## Consequences + +- `StreamSubjectResolver` and the snapshot key resolution surface gain a + required `Tenant` parameter -- a breaking change for existing resolver + implementations, mitigated by the single-static-tenant migration path. +- `JetStreamStoreError` gains a tenant-scope-mismatch variant that existing + callers must add to their match arms. +- Operators gain the option of per-tenant physical isolation, which adds + operational surface: more streams and buckets to provision, monitor, and + retention-manage. This composes with, and multiplies the scope of, + retention and compaction (see the companion retention ADR): per-tenant + buckets mean per-tenant watermark computation instead of one. +- Gets harder: any code that constructs a stream id or resolver ad hoc, + without going through the `Tenant`-aware path, must be updated -- which is + every current caller, since "append a string prefix" is the entire + mechanism today. + +## References + +- [ADR#0013: Origin Stream Sequence Header](./0013-origin-stream-sequence-header.md) +- [ADR#0023: Secret Management and Key Custody on OpenBao behind a Platform Secrets Service](./0023-secret-management-and-key-custody-direction.md) +- [ADR#0026: Command Authorization Principal and Authorizer Hook for Decider Execution](./0026-command-authorization-principal.md) +- [Event Metadata](../architecture/event-metadata.md) diff --git a/docs/adr/0028-decider-admission-control-and-backpressure.md b/docs/adr/0028-decider-admission-control-and-backpressure.md new file mode 100644 index 000000000..10ec3b91d --- /dev/null +++ b/docs/adr/0028-decider-admission-control-and-backpressure.md @@ -0,0 +1,146 @@ +--- +number: "0028" +slug: decider-admission-control-and-backpressure +status: draft +date: 2026-07-15 +--- + +# ADR#0028: Admission Control for Decider Command Execution + +## Context + +`trogon_decider_wasm_runtime::engine::WasmDeciderEngine::new_store` creates a +fresh `wasmtime::Store` for every command execution, with a fixed +per-store memory ceiling (`DEFAULT_MAX_MEMORY_BYTES`, 64 MiB) and a fuel +budget consumed per guest export call. Both +`WasmCommandExecution::execute` implementations (with and without a snapshot +store) call `engine.new_store()` unconditionally at the top of every +execution, with no check on how many stores already exist concurrently. The +per-store limiter bounds one store's worst case; it says nothing about how +many worst cases can exist at once. A burst of concurrent WASM-routed +commands can pin `N x 64 MiB` of linear memory well before any individual +guest ever hits its own limiter. + +`DeciderRegistry` routes command types to an `Arc` and is, +by its own doc comment, "built once at startup... shared read-only across +command executions" -- it holds no state about in-flight work and has no +natural place to track or reject on load. + +The native path is equally unbounded: nothing in +`trogon_decider_runtime::execution::CommandExecution::execute` gates how many +concurrent executions run against the same event store. Every command +execution, native or WASM, is accepted and run immediately, with no host-level +mechanism to shed load when concurrency spikes. + +## Decision + +### 1. A configurable admission limiter at the execution layer + +Introduce an `AdmissionLimiter`-shaped hook (a bounded-concurrency permit, +semaphore-shaped) acquired at the top of `CommandExecution::execute` and +`WasmCommandExecution::execute`, released on completion (RAII permit), before +any I/O or wasmtime work begins. It sits at the execution layer, not inside +`DeciderRegistry` or `WasmDeciderEngine`: the registry is deliberately +stateless and shared read-only, and making every `route()` lookup also +mutate a shared counter would add contention to what is today a lock-free +lookup and conflate routing with scheduling. `WasmDeciderEngine` is a +process-global wasmtime configuration wrapper, not a scheduler, and a limiter +placed only there would say nothing about the native path, which is not +wasmtime-based at all. The execution layer is the one place both paths +already funnel every command through. + +### 2. A distinguishable overloaded/retry error + +Add an `Overloaded` variant to `CommandError` and `WasmCommandError`, +parallel to the existing phase-tagged variants (`Decide`, `Evolve`, +`ReadStream`, `Append`, and so on). A caller can pattern-match it apart from +a domain rejection (`Decide`) or an infrastructure failure (`Append`) and +translate it into a retry-with-backoff or a shed-load response. Rejection is +immediate, not queued: an admission limiter that queues without a bound just +relocates the same memory pressure into the queue and trades a fast, +distinguishable error for hidden added latency. + +### 3. Interaction with the WASM per-session memory ceiling + +The admission limit and `WasmEngineConfig::max_memory_bytes` answer two +different questions and are sized together, not against each other: the +memory ceiling bounds one store's worst case, the admission limit bounds how +many worst cases can exist at once. `admission_limit x max_memory_bytes` is +the real worst-case host memory attributable to WASM guest execution, and +that product is what capacity planning budgets against, not either number in +isolation. + +## Alternatives Considered + +### Rely solely on the per-store memory limiter and fuel budget + +Rejected. They bound one store's damage, not aggregate concurrent damage. A +burst of legitimate, individually well-behaved commands can still exhaust +host memory purely by count, with every store staying under its own limit. + +### Bound concurrency inside `DeciderRegistry` + +Rejected. The registry is explicitly documented as built once and shared +read-only; making it track admission state adds write contention to a lookup +path and conflates command routing with scheduling, two concerns that should +stay independently testable. + +### Bound concurrency only inside `WasmDeciderEngine` + +Rejected as the sole location. The engine is process-global and per-module; a +limiter placed only there cannot distinguish "no room for any WASM command" +from "no room for this specific command type," and it does nothing for the +native `CommandExecution` path, which is not wasmtime-based and needs the +same protection. An engine-scoped limiter remains a reasonable additional +layer specifically for guest store count, but not a replacement for a limiter +at the execution layer that covers both paths uniformly. + +### Unbounded internal queueing instead of rejecting + +Rejected as the default. Queueing without a bound hides the same failure +mode behind added latency instead of surfacing it as a fast, distinguishable +error; it also gives callers (schedulers, gateway dispatch) no signal to +shed or retry, unlike this codebase's existing fail-loudly posture +([ADR#0017](./0017-aauth-agent-authentication.md), +[ADR#0023](./0023-secret-management-and-key-custody-direction.md)). + +## Non-Goals + +- Specifying the limiter's exact algorithm (token bucket, semaphore, leaky + bucket). Only that one exists at a defined layer with a defined error + contract. +- Per-tenant fairness or quality-of-service scheduling. A global or + per-command-type bound is in scope; weighting by tenant is a follow-on + decision that would compose with + [ADR#0027](./0027-decider-multi-tenancy-primitive.md)'s tenant value + object, but that composition is left open here. +- Bounding NATS-level publish/consume throughput. Only host-side execution + concurrency for command dispatch is in scope. +- Changing wasmtime fuel or memory defaults themselves. The admission + limiter is additive to those existing knobs, not a replacement for them. + +## Consequences + +- Every command execution path gains a new failure mode (rejected for lack + of admission), a breaking addition to `CommandError`/`WasmCommandError` + that existing callers must handle, the same category of change as the + authorization variant in + [ADR#0026](./0026-command-authorization-principal.md). +- Legitimate bursts are slowed or explicitly rejected rather than silently + degrading the whole host, but only if callers actually implement retry or + backoff against the new error; without that, callers just see more + failures with no functional improvement. +- Capacity planning becomes an explicit exercise + (`admission_limit x max_memory_bytes` for WASM); a default that is not + tuned against real traffic can be too conservative (throttling legitimate + load) or too permissive (not actually protecting the host). +- Gets harder: test suites that fire many concurrent commands in a tight + loop, including this crate's own WASM execution tests, may need to account + for the limiter rather than assume unbounded concurrency. + +## References + +- [ADR#0017: AAuth Agent Authentication over a Trogon NATS PoP Binding](./0017-aauth-agent-authentication.md) +- [ADR#0023: Secret Management and Key Custody on OpenBao behind a Platform Secrets Service](./0023-secret-management-and-key-custody-direction.md) +- [ADR#0026: Command Authorization Principal and Authorizer Hook for Decider Execution](./0026-command-authorization-principal.md) +- [ADR#0027: Tenant Value Object for Decider Stream and Snapshot Resolution](./0027-decider-multi-tenancy-primitive.md) diff --git a/docs/adr/0029-decider-retention-and-truncation-watermark.md b/docs/adr/0029-decider-retention-and-truncation-watermark.md new file mode 100644 index 000000000..14ffdf52f --- /dev/null +++ b/docs/adr/0029-decider-retention-and-truncation-watermark.md @@ -0,0 +1,154 @@ +--- +number: "0029" +slug: decider-retention-and-truncation-watermark +status: draft +date: 2026-07-15 +--- + +# ADR#0029: Snapshot-Derived Retention Watermark for Decider Streams + +## Context + +Nothing in `trogon-decider-nats` computes the minimum JetStream stream +sequence still needed for a logical stream. A snapshot's recorded +`StreamPosition` is, in principle, exactly the value that makes earlier +events safe to discard for that stream: both `CommandExecution` and +`WasmCommandExecution` resume replay from `ReadFrom::after(snapshot_position)` +after loading a snapshot, so nothing before the newest snapshot's position is +ever read again in the normal execution path. But nothing aggregates this +across the snapshot ids and checkpoints (`snapshots.checkpoint.*` keys, +tracked via `NatsSnapshotConfig`) that share the one physical stream and KV +bucket, and `JetStreamStore` never calls a JetStream purge or trim operation +at all -- it only reads stream info and raw messages, publishes, and reads or +writes KV entries. Streams and KV history both grow unboundedly by default. + +Because every logical stream is a subject on one shared physical JetStream +stream, any retention action is a shared, blunt instrument unless it targets +a specific subject. A stream-wide `max_age`/`max_bytes`/`max_msgs` limit +cannot know that one logical stream's oldest event is still needed while +another's is not; tuning it for a fast-moving stream would silently evict a +slow-moving stream's un-snapshotted history. + +[ADR#0013](./0013-origin-stream-sequence-header.md) already separates +physical JetStream stream sequence (authoritative for checkpoints, +high-water marks, and optimistic concurrency) from +`Trogon-Origin-Stream-Sequence` (provenance metadata only, present only on +restored or migrated events). Any retention design has to key off physical +positions, the same ones snapshots already record, never off origin +provenance -- a stream that has been restored has origin and physical +positions that diverge by design, and origin sequence is absent on the +common case of an ordinary append. + +## Decision + +### 1. A safe-truncation watermark API + +Define a watermark computation, `MinimumRequiredSequence` (or equivalent), +that reports the minimum physical stream sequence that must be retained for +a given logical stream: `min(snapshot.position)` across every outstanding +snapshot and checkpoint that currently exists for that stream id, using the +`StreamPosition` values `SnapshotWrite`/`Snapshot` already record. Everything +strictly below that minimum is safe to discard for that logical stream, +because no execution path resumes from earlier than the newest retained +snapshot's position. The API is a read-only query: it reports a watermark +per stream id (or in aggregate across a physical stream), and computing it +has no side effect. + +### 2. Truncation is invoked by an operator or a scheduled job, never by the store + +`JetStreamStore` and the snapshot store expose the watermark as something a +caller queries; no method on either issues a JetStream purge or trim call as +a side effect of a normal read, append, or snapshot write. A separate, +explicitly invoked maintenance job (an operator command, a scheduled task, or +an admin action) reads the watermark and issues the purge. This keeps +"decide whether to truncate," an operational and reviewable decision, apart +from "execute a command," the store's actual job -- the same separation +[ADR#0023](./0023-secret-management-and-key-custody-direction.md) draws +between routine operation and an operator-invoked, out-of-band action. + +### 3. Recommended retention policy posture + +Default to no automatic stream-wide `max_age`/`max_msgs` limit on the shared +physical stream: a stream-wide limit cannot know which logical streams still +need history no snapshot has covered yet. Retention is watermark-driven, +per-subject purge instead, run on a schedule that trails behind snapshot +cadence -- a job that fails to run for a while only means storage grows, never +that a needed event disappears prematurely. A conservative stream-wide +`max_age` far longer than any expected snapshot interval is a reasonable +backstop, but it is a safety net, not the retention mechanism. + +### 4. Interplay with ADR#0013 physical positions + +The watermark is computed and purge is executed exclusively in terms of +physical JetStream stream sequence, never `Trogon-Origin-Stream-Sequence`. A +restored or migrated stream is truncated using its current physical sequence +like any other stream; origin provenance headers on events that remain after +truncation are unaffected and continue to describe where those events +originally lived. + +## Alternatives Considered + +### Let JetStream's own stream-level retention policy do the whole job + +Rejected as the sole mechanism. `max_age`/`max_msgs`/`max_bytes` operate on +the whole physical stream and cannot know that one logical stream's oldest +event is still needed while another's is not. + +### Auto-truncate whenever a new snapshot is written + +Rejected. It turns every snapshot write into an operation with unbounded +downstream cost (a purge call), removes the operator's ability to pause, +review, or rate-limit truncation independently of command traffic, and makes +a bug in watermark computation immediately destructive instead of surfacing +first as an observable, reviewable report. + +### Key the watermark off `Trogon-Origin-Stream-Sequence` + +Rejected. That header is provenance metadata only +([ADR#0013](./0013-origin-stream-sequence-header.md)), deliberately not +authoritative for checkpoints or optimistic concurrency, and absent on the +common case of an ordinary append. Using it would either break on +non-restored streams or misjudge safety on restored ones, where physical and +origin positions diverge by design. + +## Non-Goals + +- Implementing the purge/trim job itself -- its scheduling, failure handling, + or authorization. Only the watermark API it consumes and the + invocation-boundary rule (operator/job, never the store) are in scope. +- A full KV bucket history-depth (compaction) design. Noting that KV + revision history exists and grows is in scope; deciding how many + historical revisions per snapshot key to retain is follow-up work once a + stream-side watermark exists to model it against. +- Changing snapshot frequency or policy (`SnapshotPolicy`, + `FrequencySnapshot`). Retention consumes whatever cadence a decider already + chose. +- Per-tenant retention policy differences. Composes with, but does not + depend on, [ADR#0027](./0027-decider-multi-tenancy-primitive.md). + +## Consequences + +- Operators gain an explicit, reviewable maintenance surface instead of + unbounded growth, and an operational responsibility that has no effect + until a deployment actually wires in and schedules the job. +- The watermark's correctness matters more than most read paths in this + crate: undercounting outstanding snapshots or checkpoints is a data-loss + bug, not a performance bug, and needs test coverage against every snapshot + type a decider can register, not just the common case. +- Gets harder: adding a new snapshot type or checkpoint kind to a decider + now carries a second obligation beyond storage -- it must be discoverable + by whatever aggregates "all outstanding snapshots for this stream," or the + watermark silently ignores it and a purge could delete events it still + needs. +- Because retention purges by subject against one shared physical stream, + purge operations for many logical streams still serialize against that one + stream's admin surface. Operators truncating frequently across many + tenants or streams should expect purge throughput, not just storage size, + to become a capacity question. + +## References + +- [ADR#0013: Origin Stream Sequence Header](./0013-origin-stream-sequence-header.md) +- [ADR#0023: Secret Management and Key Custody on OpenBao behind a Platform Secrets Service](./0023-secret-management-and-key-custody-direction.md) +- [ADR#0027: Tenant Value Object for Decider Stream and Snapshot Resolution](./0027-decider-multi-tenancy-primitive.md) +- [NATS JetStream Source and Mirror Streams](https://docs.nats.io/nats-concepts/jetstream/source_and_mirror) diff --git a/docs/adr/index.md b/docs/adr/index.md index 83255f1ec..88d01c038 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -29,3 +29,7 @@ future implementation work. - [ADR#0021: Typed Decode over Passthrough Forwarding](./0021-typed-decode-over-passthrough-forwarding.md) - [ADR#0022: Canonical ACP Method Vocabulary in the NATS Layer (Rejected)](./0022-canonical-acp-wire-methods-on-nats.md) - [ADR#0023: Secret Management and Key Custody on OpenBao behind a Platform Secrets Service](./0023-secret-management-and-key-custody-direction.md) +- [ADR#0026: Command Authorization Principal and Authorizer Hook for Decider Execution](./0026-command-authorization-principal.md) +- [ADR#0027: Tenant Value Object for Decider Stream and Snapshot Resolution](./0027-decider-multi-tenancy-primitive.md) +- [ADR#0028: Admission Control for Decider Command Execution](./0028-decider-admission-control-and-backpressure.md) +- [ADR#0029: Snapshot-Derived Retention Watermark for Decider Streams](./0029-decider-retention-and-truncation-watermark.md) diff --git a/docs/architecture/decider.md b/docs/architecture/decider.md new file mode 100644 index 000000000..56f2b7f26 --- /dev/null +++ b/docs/architecture/decider.md @@ -0,0 +1,459 @@ +# Decider Platform + +The decider platform is this workspace's event-sourcing runtime: a `Decider` trait for +domain decision logic, a native execution boundary that replays and appends JetStream +streams, a WASM host that runs the exact same decision logic compiled to a component, and +shared testing/observability tooling so both paths stay provably in sync. This is the guide +to read first; it links out to the ADRs and per-crate READMEs for anything it only +summarizes. + +## The `decide`/`evolve`/`initial_state` cycle + +Every decider is a plain Rust type implementing `trogon_decider::Decider` +(`crates/trogon-decider/src/lib.rs`): + +```rust +pub trait Decider: Sized { + type StreamId: ?Sized; + type State; + type Event; + type DecideError: std::error::Error; + type EvolveError: std::error::Error; + + const WRITE_PRECONDITION: Option = None; + + fn stream_id(&self) -> &Self::StreamId; + fn initial_state() -> Self::State; + fn evolve(state: Self::State, event: &Self::Event) -> Result; + fn decide(state: &Self::State, command: &Self) -> Result, Self::DecideError>; +} +``` + +`initial_state` seeds state with no events applied. `evolve` folds one stored event into +state and is the only place state changes. `decide` evaluates a command against the current +state and returns a `Decision`, never mutating anything itself. When a rejection crosses the +WASM boundary, the bridge surfaces it with the constant `"rejected"` code on the WIT +`domain-error` and carries the typed error's causal chain in its `details` pairs. + +`WRITE_PRECONDITION` (`crates/trogon-decider/src/write_precondition.rs`) is the optional +concurrency guard applied when the decided events are appended: + +```rust +pub enum WritePrecondition { + Any, // append regardless of current stream state + StreamExists, // append only if the stream already has events + NoStream, // append only if the stream is empty (first writer wins) +} +``` + +### `Decision::Events` vs `Decision::Act` + +`decide` returns a `Decision` (`crates/trogon-decider/src/decision.rs`): + +```rust +#[non_exhaustive] +pub enum Decision where C: Decider { + Events(Events), + Act(Act), +} +``` + +`Decision::events(...)` (or the single-event `Decision::event(...)`) is the common case: a +flat, already-final batch of events. `Decision::act()` starts an `ActBuilder` instead, for +decisions that need to observe the state produced by an earlier step before deciding the +next one: + +```rust +Decision::act() + .execute(|state, command| { /* returns Decision or Result, C::DecideError> */ }) + .execute(|state, command| { /* sees the state the previous step's events would produce */ }) +``` + +Each `.execute` call returns a new typestate `ActChain` (`S: Steps`, built from +`First`/`Then` link types); these link types and `ActChain`/`ActRun`/`Steps` are +`#[doc(hidden)]` implementation detail. Converting the finished chain into a +`Result, C::DecideError>` boxes the whole plan exactly once into an `Act`, so +`Decision` can hold either form as one uniform type. `evaluate_decision` +(`crates/trogon-decider/src/decision.rs`, `#[doc(hidden)]`) is the shared entry point both the +native runtime and the WASM guest bridge call: `C::decide` then, for an `Act`, runs each step +in turn, folding that step's events via `C::evolve` before handing the resulting state to the +next step. This is why `Decision::Act` behaves identically on both paths: neither path +reimplements the stepping logic. + +## The native path: `CommandExecution` + +`trogon-decider-runtime`'s `CommandExecution<'a, E, C, S, G>` +(`crates/trogon-decider-runtime/src/execution.rs`) is the runtime boundary that applies one +command to one stream: read, replay, decide, append. Build one with `CommandExecution::new` +and configure it with builder methods before calling `execute`: + +```rust +CommandExecution::new(&event_store, &command) + .with_write_precondition(precondition) // ignored if C::WRITE_PRECONDITION is set + .with_headers(headers) + .with_event_id_generator(generator) + .with_snapshot(snapshot_store) // moves to the snapshot-enabled type state + .with_task_runtime(scheduler) + .with_snapshot_failure_policy(policy) + .execute() + .await +``` + +There are two `execute` methods, selected by type state (whether `.with_snapshot(...)` was +called): + +- **Without snapshots**: if the decider declares `WRITE_PRECONDITION = Some(NoStream)`, the + stream isn't read at all and the command is decided straight against `C::initial_state()`. + Otherwise the stream is read from the beginning (`ReadFrom::Beginning`), folded through + `evolve`, decided, and appended. +- **With snapshots**: a snapshot is read first. On a snapshot the configured + `SnapshotFailurePolicy` cannot trust, the policy decides `Fail` or `DiscardAndReplay`; a + discarded snapshot falls back to a full replay from the beginning. Otherwise only the + stream events *after* the snapshot's position are read (`ReadFrom::after(position)`). A + snapshot whose position is ahead of the stream's current position is also routed through + the failure policy. After decide and append succeed, a snapshot write is scheduled: always, + if a bad snapshot was just discarded and overwritten with a trustworthy one; otherwise only + when the configured `SnapshotPolicy::decide_snapshot` returns `SnapshotDecision::Take`. + +`CommandError` normalizes failures by phase +(`Decide`, `Evolve`, `ReadSnapshot`, `ReadStream`, `Append`, `EventType`, `EventEncode`, +`DecodeEvent`, `SnapshotAheadOfStream`, `ReadAfterOverflow`) while preserving each phase's +concrete source error type. + +### Snapshot policy and failure recovery + +`SnapshotPolicy` decides whether to take a snapshot after a successful append +(`decide_snapshot(context: DecideSnapshot<'_, C>) -> SnapshotDecision`). Two implementations +ship: `NoSnapshot` (never) and `FrequencySnapshot` (takes one once at least `frequency` +events have been read or appended since the last snapshot). `CommandSnapshotPolicy: Decider` +lets a decider declare its own default policy via an associated `SNAPSHOT_POLICY` const, so +callers build a configured `Snapshots` without repeating the policy at every call site. + +`SnapshotFailurePolicy` decides how to react to a snapshot execution +cannot trust: `FailOnSnapshotFailure` (the default, fails the command) or +`DiscardAndReplaySnapshotFailure` (discards the bad snapshot and replays from the beginning +of the stream instead of failing). + +Scheduled snapshot writes run through a `SnapshotTaskScheduler` (`fn schedule`, plus a +`drain` future that defaults to resolving immediately). `TokioSnapshotTaskScheduler` is +fire-and-forget: its `drain` never actually waits. `DrainableSnapshotTaskScheduler` tracks +in-flight writes so a host can `drain().await` outstanding snapshot writes before shutdown. + +### Headers + +Event headers are operational metadata (correlation, tenancy, causation, transport +routing), distinct from event payloads, which are the canonical domain facts. The runtime +does not derive required headers from commands or events through a generic callback; +callers set headers explicitly via `CommandExecution::with_headers`. See +[Event Metadata](./event-metadata.md) for the full policy, and +[ADR#0013](../adr/0013-origin-stream-sequence-header.md) for the one reserved header name +this repository defines today, `Trogon-Origin-Stream-Sequence`, used only for provenance +when an event is republished into a different stream from the one it originally lived in. +It is never used for checkpoints, positions, or concurrency; those always use the stream's +current sequence. + +## JetStream storage layer (`trogon-decider-nats`) + +`trogon-decider-nats` implements the native runtime's storage traits (stream read/append, +snapshot read/write) against NATS JetStream. Per [ADR#0002](../adr/0002-rust-crate-boundaries.md), +this crate is the adapter layer: it knows how to store and replay a stream, not how to +`decide`/`evolve`. + +### One physical stream, subject per logical stream + +Every decider's events live in **one physical JetStream stream**. A caller-supplied +`StreamSubjectResolver` maps a domain stream identifier to the subject that stores +it (`resolve_subject_state`, returning a `SubjectState { subject, current_position }`), so +each logical stream gets its own subject within that one physical stream: + +```text +physical JetStream stream "orders-events" + ├─ subject "orders.events.order-1" (logical stream "order-1") + ├─ subject "orders.events.order-2" (logical stream "order-2") + └─ subject "orders.events.order-3" (logical stream "order-3") +``` + +`ensure_stream`/`ensure_bucket` (`provision.rs`) idempotently create-or-open the physical +stream and the KV bucket used for snapshots: a create that fails because the resource +already exists resolves via a get instead of surfacing an error, so concurrent callers racing +to provision the same resource both succeed. + +### Atomic multi-event append + +`append_stream` (`stream_store.rs`) publishes a batch of events to one subject atomically +using three headers on the underlying JetStream messages: `Nats-Batch-Id` and +`Nats-Batch-Sequence` on every message in the batch, `Nats-Batch-Commit` on the final +message, and (when appending is conditional) the expected-last-subject-sequence guard on the +first message. The final message's stream sequence becomes the returned `StreamPosition`. + +### Ordered, subject-filtered replay + +Replay (`stream_store/replay.rs`) uses an ordered JetStream consumer scoped with +`filter_subject` and `DeliverPolicy::ByStartSequence { start_sequence }`, reading only the +target logical stream's own messages in order rather than scanning every physical sequence. +The consumer is self-healing for most disconnects; residual failure kinds are retried by +recreating the consumer from the last successfully processed sequence, bounded by +`ReplayRetryPolicy`: 5 attempts, 100ms base delay, 5s max delay, exponential backoff between +attempts. + +### KV-backed snapshots + +Snapshot storage (`snapshot_store.rs`) is a set of free functions generic over a JetStream KV +bucket (`read_snapshot`, `write_snapshot`, `read_checkpoint`, `write_checkpoint`, +`maybe_advance_checkpoint`, `persist_snapshot_change`), not a dedicated struct. +`NatsSnapshotConfig` carries the checkpoint key name; `SnapshotChange` is either an +`Upsert` or a `Delete` mutation applied atomically alongside the checkpoint update. + +## Read-side primitives: `Projector` and `Processor` + +`trogon-decider-nats` also ships two generic read-side primitives, so consumers of decider +streams don't hand-roll catch-up or redelivery logic: + +- **`Projector`** drives catch-up for one subject-filtered projection: given a + `ProjectionCheckpointStore` (load/save a `CheckpointSequence`) and a `ProjectionApply` + (apply one event, return the checkpoint to record for it), `catch_up` replays events in + stream order up to the tail observed when the call started, saving the checkpoint after + each one. Use `Projector` when you're building a read model or side-effect projection that + needs to resume from wherever it last left off. +- **`Processor`** drives a durable pull consumer's message loop for one + `MessageHandler` (`handle` returns a `HandlerVerdict`: `Ack`, `Retry`, or `Poison`). + `RedeliveryPolicy` bounds retries (default: 5 max deliveries, 200ms base delay, 30s max + delay, exponential backoff) before a message is poisoned and `on_poison` is called with a + `PoisonReason` (`Verdict`, `RedeliveryExhausted`, `HandlerError`, or `Panic`). Use + `Processor` when you need at-least-once, ack/nak/term message processing against a durable + consumer, not a resumable full-stream projection. + +`CheckpointSequence` wraps the underlying `u64` stream sequence; `CheckpointSequence::NONE` +(value `0`) means no event has been applied yet, distinct from having processed sequence 1. +`next_from_sequence()` returns the sequence a catch-up should resume from (the checkpoint's +sequence plus one). + +## The WASM path + +The same `Decider` logic can be compiled to a WASM component and executed by +`trogon-decider-wasm-runtime` instead of running natively. Both paths share the same +`decide`/`evolve` semantics (via `evaluate_decision`) and the same event codec, so a decider +behaves identically whether it runs natively or as a guest. + +### `export_decider!` and the WIT contract + +`export_decider!` (`trogon-decider-guest-macros`) generates the component glue for one or +more commands that share a `Decider::State` and `Decider::Event`: + +```rust +export_decider!( + CreateSchedule { + type_url = CREATE_SCHEDULE_TYPE_URL, + proto = v1::CreateSchedule, + module = "scheduler.schedules", + version = "0.1.0", + state_schema_version = SCHEDULES_STATE_SCHEMA_VERSION, + write_precondition = no_stream, + }, +); +``` + +It expands to a hidden module containing `wit_bindgen::generate!` bindings, a `Component` +unit struct implementing the WIT `Guest` trait (`descriptor()`, `stream_id(command)`), and a +`Session` struct (`RefCell`) implementing `GuestSession`: `new` loads a snapshot or +builds initial state, `evolve` folds events and mutates the `RefCell`, `decide` only borrows +state (it does not fold its own output back in), and `snapshot` encodes current state. The +macro resolves the WIT contract via `trogon_decider_wit::WIT_DIR`, an absolute path baked +into `trogon-decider-wit` at that crate's own compile time, so a component crate can live at +any directory depth relative to it. + +The contract itself lives in `trogon-decider-wit`'s `wit/world.wit` (package +`trogon:decider@0.2.0`, world `decider`, interface `handler`): `descriptor()` returns a +`module-descriptor { name, version, commands }`; `stream-id(command)` resolves a +`command-envelope` to its target stream; the `session` resource exposes +`evolve(events)`, `decide(command) -> result, decide-error>`, and +`snapshot() -> option>`. `domain-error { code, message, details }` carries a stable +code, a human-readable message, and the error's source chain as ordered key/value pairs, so +detail that a single `Display` line would otherwise erase survives the WIT boundary. +`trogon-decider-wit` compiles this contract for both sides: `guest` (feature `guest`, via +`wit_bindgen::generate!`, for `wasm32` components) and `host` (feature `host`, via +`wasmtime::component::bindgen!`, for non-`wasm32` runtimes), re-exporting the same +WIT-derived types from both. + +### Engine budgets and deadlines + +`WasmDeciderEngine`/`WasmEngineConfig` (`trogon-decider-wasm-runtime/src/engine.rs`) configure +a shared wasmtime `Engine` with fuel metering, epoch-based interruption, the pooling +allocator, and per-store resource limits, built once per process and shared across every +loaded module: + +| Budget | Default | Purpose | +| --- | --- | --- | +| `fuel_per_call` | 10,000,000 | Instructions a single `decide`/`evolve`/`snapshot` call may execute before trapping `OutOfFuel`. | +| `max_memory_bytes` | 64 MiB | Ceiling on a store's linear-memory growth. | +| `max_table_elements` | 8,192 | Ceiling on any table a session's store may create. | +| `max_instances_per_session` | 8 | Ceiling on core module instantiations per session's store. | +| `max_tables_per_session` | 6 | Ceiling on tables per session's store. | +| `max_memories_per_session` | 4 | Ceiling on memories per session's store. | +| `max_concurrent_sessions` | 256 | Pooling-allocator warm-slot ceiling. | +| `epoch_tick_interval` | 50ms | Cadence of the engine's background epoch ticker. | +| `epoch_ticks_per_call` | 40 | Ticks a call may run before being interrupted, i.e. a 2 second wall-clock budget per call at the default cadence. | + +Fuel bounds instructions, not time; the epoch ticker is the wall-clock backstop for a guest +that is still executing but too slowly. A `WasmDeciderModule` is loaded once (`load`), which +compiles the component, structurally enforces zero imports by instantiating it against an +empty `wasmtime::component::Linker` (the same requirement `trogon-decider-sim`'s +`assert_zero_imports` checks for, via `wasm-tools` instead), caches the resulting `DeciderPre` +for cheap per-command instantiation, and probes the guest's `descriptor()` export once. + +### Execution ordering: decide, fold, then snapshot + +`WasmCommandExecution` mirrors the native `CommandExecution` boundary against the WIT +`session` resource. Guest calls run via `spawn_guest`, which wraps +`tokio::task::spawn_blocking` so a guest's fuel- and epoch-bounded call never occupies the +async executor for its duration. + +The guest's generated `decide` export only reads session state; it never folds its own +newly decided events back into it (`evolve` mutates the session's `RefCell`, `decide` +only borrows it). A snapshot taken immediately after `decide` would therefore silently drop +that command's own events. The snapshot-enabled execution path calls `decide`, then +`fold_decided_events` (which replays those events into the session via `evolve`), then +`take_snapshot`, in that order, so a session resumed from that snapshot ends up in the +same state a full replay would produce. The no-snapshot path skips the fold entirely: no +snapshot will observe that session again, so folding would only burn guest fuel for nothing. + +`WasmCommandError` distinguishes a guest call that trapped for any other reason (`Trap`) from +one that exceeded its wall-clock epoch deadline (`DeadlineExceeded`), classified by +downcasting the trap to `wasmtime::Trap::Interrupt`. + +### Snapshot failure recovery mirrors the native path + +`WasmCommandExecution::with_snapshot_failure_policy` mirrors native's +`SnapshotFailurePolicy`: a snapshot the host cannot trust, either because reading it failed or +because it claims a position ahead of the stream, is routed through a +`WasmSnapshotFailurePolicy` before either failure is allowed to fail the +command. `FailOnSnapshotFailure` (the default) keeps today's behavior; +`DiscardAndReplaySnapshotFailure` discards the untrusted snapshot and replays the stream from +the beginning instead, exactly as `CommandExecution` does natively. Both policy types are +reused directly from `trogon-decider-runtime`, since neither carries any decider-specific +state. + +The WASM boundary has no typed `Decider`, so the policy's input is a `WasmSnapshotFailureContext` +carrying whatever identity the execution actually has in hand at that point instead of a +command reference: the module's name and version, the command's wire type URL, the resolved +stream id, and the failure itself (reused as native's `SnapshotFailure`). + +Native forces the post-append snapshot write unconditionally when a bad snapshot was just +discarded, so a `SnapshotPolicy` that would otherwise skip cannot let the bad snapshot linger. +The WASM path needs no equivalent branch: it has no host-side `SnapshotPolicy` to skip with in +the first place, since a snapshot is written whenever the guest's `snapshot()` export returns +one at all. + +### Registry rollout: swapping modules without a restart + +`DeciderRegistry` routes command types to the `WasmDeciderModule` that declared them, built +once via `DeciderRegistryBuilder` (which rejects command-type collisions across modules). +`DeciderRegistryHandle` wraps a `DeciderRegistry` behind a `RwLock>` so +it can be updated while command executions are in flight: + +```text + ┌──────────────────────────┐ + readers ────► │ RwLock> │ ◄──── activate(module) / retire(command_type) + └──────────────────────────┘ +``` + +`activate(module)` routes every command type the module declares to it, replacing whatever +module previously owned each route; `retire(command_type)` removes one route. Both compute +the entire next routing table first and take the write lock only long enough to install it +with one assignment, so a module declaring more than one command type is activated or +retired as a unit: a reader never observes some of its command types on the new module +while a sibling command type from the same module is still on the old one. A caller resolves +a module once per command dispatch and keeps using that resolved `Arc`; a later `activate` +or `retire` never reaches back into an execution already running against a module it +resolved earlier. + +`WasmSnapshotId` folds a module's identity into every snapshot id it produces: +`{module_name}@{module_version}/{stream_id}`. Routing a command type from module version `v1` +to `v2` via `activate` does not migrate or invalidate `v1`'s snapshots: `v2`'s snapshot id for +a given stream is simply a different string, so `v2`'s first command against a stream `v1` +had already snapshotted finds nothing under `v2`'s id and falls back to a full replay from +the beginning, the same fallback taken for a stream that was never snapshotted at all. That +replay reconstructs the same guest state a snapshot would have, just without resuming from a +saved position. No explicit migration or invalidation step is needed; the version bump +folded into the snapshot id is enough on its own. + +## Testing story + +Four layers, from unit-level to conformance-level: + +- **`TestCase`** (`trogon-decider/src/testing.rs`) is a given/when/then typestate builder for + unit-testing a native `Decider` directly: seed a `History`, issue a command, + assert the resulting `Decision` via a `Then`-style expectation (event equality, a + rejection, or a specific error code), and optionally assert the post-decision state. +- **`trogon-decider-sim`** is the native-vs-wasm parity harness. `SimHost`/`SimInstance` + (`host.rs`) load a compiled component and run every guest call against the same resource + budget `WasmDeciderEngine` applies in production, so a decider that would trap in + production traps the same way in tests instead of running unbounded. `ScenarioIr`/ + `WireEnvelope` (`ir.rs`) express a decider-agnostic given/when/then scenario in wire form + (type URL plus encoded payload) that both a wasm run (`ScenarioIr::run_wasm`) and a native + run (`native.rs`'s `run_native`, reusing the exact `trogon-decider-guest-sdk` codec + functions the guest bridge expands into) can execute. `SimScenario` (`scenario.rs`) is the + fluent given/when/then builder used directly against a `SimInstance`, supporting multiple + chained steps against one open session. `assert_parity` (`parity.rs`) runs one scenario + through both runners and returns an error on the first divergence between their outcomes, + catching codec, codegen, or WIT-level drift that each runner's own pass/fail would miss. + `SimFixture` (`fixture.rs`, feature `test-support`) holds checked-in compiled `.wasm` + fixtures for integration tests. +- **`decider-test` CLI** (`cli/trogon-decider-test`) runs a YAML conformance suite against a + compiled component with no Rust required. A `Suite` declares scenarios (`Scenario`, each + either a legacy single `when`/`then` pair or an ordered `steps: Vec` list run against + one session) and the event type URLs the decider is declared to produce. Strict mode + (default on, disabled with `--no-strict`) fails the run if any declared command or event + type has zero coverage across every scenario's `given`/`when`/`then.events`; detection and + reporting always run, `--no-strict` only downgrades the failure to a stderr warning. +- **`assert_zero_imports`** (`trogon-decider-sim/src/import_check.rs`) checks a compiled + component declares no imports via `wasm-tools component wit`, the same zero-import + requirement `WasmDeciderModule::load` enforces structurally at load time with an empty + `Linker`. + +## Observability + +Per [ADR#0008](../adr/0008-opentelemetry-observability.md), the decider platform is +instrumented against `otel/semconv/registry/decider.yaml`'s `registry.trogonai.decider` +attribute group. All decider telemetry is `stability: development`. + +Attributes: `command_type`, `module_name`, `module_version`, `write_precondition` +(`any`/`stream_exists`/`no_stream`/`at`), `decision_outcome` +(`decided`/`rejected`/`faulted`), `snapshot_outcome` +(`hit`/`miss`/`discarded_read_failure`/`discarded_ahead_of_stream`/`failed`), +`snapshot_write_success`, `guest_phase` (`instantiate`/`replay`/`decide`/`snapshot`), and +`trap_classification` (`deadline_exceeded`/`trap`). `stream_id` is defined once in +`scheduler.yaml` and reused here. + +Spans: `decider.execute_command`, `decider.read_snapshot`, `decider.write_snapshot`, +`decider.append_stream`, `decider.replay_stream`, `decider.wasm.instantiate`, +`decider.wasm.replay`, `decider.wasm.decide`, `decider.wasm.snapshot`. + +Metrics: `decider.replay.events` (counter), `decider.snapshot.reads` (counter, by +`snapshot_outcome`), `decider.snapshot.writes` (counter, by `snapshot_write_success`), +`decider.snapshot.kv_read_failures` / `decider.snapshot.kv_write_failures` (counters), +`decider.append.duration` (histogram), `decider.append.conflicts` (counter), +`decider.replay.duration` (histogram), `decider.replay.retries` (counter), +`decider.wasm.execution.duration` (histogram, by `guest_phase`), +`decider.wasm.fuel.consumed` (histogram, by `guest_phase`), `decider.wasm.traps` (counter, by +`trap_classification`). + +## Related ADRs + +Accepted and reflected in the platform as built: + +- [ADR#0002: Rust Crate Boundaries](../adr/0002-rust-crate-boundaries.md): the + domain/runtime/adapter split this document follows (`trogon-decider` / + `trogon-decider-runtime` / `trogon-decider-nats`). +- [ADR#0008: OpenTelemetry Observability](../adr/0008-opentelemetry-observability.md): the + vocabulary requirement behind the Observability section above. +- [ADR#0013: Origin Stream Sequence Header](../adr/0013-origin-stream-sequence-header.md): + the `Trogon-Origin-Stream-Sequence` header described under Headers above. + +Draft, **not yet implemented**; do not treat these as current behavior: + +- [ADR#0026: Command Authorization Principal](../adr/0026-command-authorization-principal.md) +- [ADR#0027: Decider Multi-Tenancy Primitive](../adr/0027-decider-multi-tenancy-primitive.md) +- [ADR#0028: Decider Admission Control and Backpressure](../adr/0028-decider-admission-control-and-backpressure.md) +- [ADR#0029: Decider Retention and Truncation Watermark](../adr/0029-decider-retention-and-truncation-watermark.md) diff --git a/otel/semconv/registry/decider.yaml b/otel/semconv/registry/decider.yaml new file mode 100644 index 000000000..52bf8720e --- /dev/null +++ b/otel/semconv/registry/decider.yaml @@ -0,0 +1,456 @@ +groups: + # trogon-decider-runtime, trogon-decider-nats, trogon-decider-wasm-runtime: + # decider command execution, event-sourced replay/append, and WASM guest + # execution. `stream_id` is defined in scheduler.yaml and reused here as-is. + - id: registry.trogonai.decider + type: attribute_group + display_name: Decider Attributes + brief: Attributes emitted by the decider runtime, its NATS JetStream storage adapter, and its WASM guest execution engine. + attributes: + - id: command_type + type: string + stability: development + brief: Command type identifier for the decider command being processed. + examples: ["OpenAccount", "schedules.create_schedule"] + - id: module_name + type: string + stability: development + brief: Name of the WASM decider module handling the command. + examples: ["schedules"] + - id: module_version + type: string + stability: development + brief: Version of the WASM decider module handling the command. + examples: ["1.0.0"] + - id: write_precondition + type: + members: + - id: any + value: "any" + brief: Append regardless of the stream's current position. + stability: development + - id: stream_exists + value: "stream_exists" + brief: Append only if the stream already has at least one event. + stability: development + - id: no_stream + value: "no_stream" + brief: Append only if the stream has no events. + stability: development + - id: at + value: "at" + brief: Append only if the stream is still at the observed position. + stability: development + stability: development + brief: Optimistic concurrency precondition applied to a stream append. + examples: ["any", "at"] + - id: decision_outcome + type: + members: + - id: decided + value: "decided" + brief: The command was decided and its events appended. + stability: development + - id: rejected + value: "rejected" + brief: The command was rejected by domain decision logic. + stability: development + - id: faulted + value: "faulted" + brief: Command execution faulted on an infrastructure or guest execution failure. + stability: development + stability: development + brief: Outcome of a command execution attempt. + examples: ["decided", "rejected"] + - id: snapshot_outcome + type: + members: + - id: hit + value: "hit" + brief: A snapshot was found and used as the replay starting point. + stability: development + - id: miss + value: "miss" + brief: No snapshot existed; state was rebuilt from the initial state. + stability: development + - id: discarded_read_failure + value: "discarded_read_failure" + brief: A snapshot read failure was discarded and replay restarted from the initial state. + stability: development + - id: discarded_ahead_of_stream + value: "discarded_ahead_of_stream" + brief: A snapshot ahead of the stream's current position was discarded. + stability: development + - id: failed + value: "failed" + brief: A snapshot problem was detected and the command failed instead of discarding the snapshot. + stability: development + stability: development + brief: Outcome of the snapshot read at the start of command execution. + examples: ["hit", "miss"] + - id: snapshot_write_success + type: boolean + stability: development + brief: Whether a decider snapshot write succeeded. + - id: guest_phase + type: + members: + - id: instantiate + value: "instantiate" + brief: Instantiating the WASM guest component. + stability: development + - id: replay + value: "replay" + brief: Replaying prior events into the guest session. + stability: development + - id: decide + value: "decide" + brief: Calling the guest's decide export. + stability: development + - id: snapshot + value: "snapshot" + brief: Reading or writing the guest session's snapshot. + stability: development + stability: development + brief: Phase of WASM guest command execution. + examples: ["instantiate", "decide"] + - id: trap_classification + type: + members: + - id: deadline_exceeded + value: "deadline_exceeded" + brief: The guest call exceeded its wall-clock epoch deadline. + stability: development + - id: trap + value: "trap" + brief: The guest call trapped for a reason other than the epoch deadline, e.g. fuel exhaustion, a memory limit, or an ABI failure. + stability: development + stability: development + brief: Classification of a WASM guest call trap. + examples: ["deadline_exceeded", "trap"] + + - id: metric.decider.replay.events + type: metric + metric_name: decider.replay.events + annotations: + code_generation: + metric_value_type: int + brief: Events replayed to rebuild decider state before a command decision. + instrument: counter + unit: "{event}" + stability: development + attributes: + - ref: command_type + requirement_level: recommended + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.snapshot.reads + type: metric + metric_name: decider.snapshot.reads + annotations: + code_generation: + metric_value_type: int + brief: Snapshot reads at the start of command execution, by outcome. + instrument: counter + unit: "{read}" + stability: development + attributes: + - ref: snapshot_outcome + requirement_level: required + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.snapshot.writes + type: metric + metric_name: decider.snapshot.writes + annotations: + code_generation: + metric_value_type: int + brief: Scheduled decider snapshot writes, by success. + instrument: counter + unit: "{write}" + stability: development + attributes: + - ref: snapshot_write_success + requirement_level: required + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.snapshot.kv_read_failures + type: metric + metric_name: decider.snapshot.kv_read_failures + annotations: + code_generation: + metric_value_type: int + brief: JetStream KV snapshot read failures in the storage adapter. + instrument: counter + unit: "{failure}" + stability: development + attributes: + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.snapshot.kv_write_failures + type: metric + metric_name: decider.snapshot.kv_write_failures + annotations: + code_generation: + metric_value_type: int + brief: JetStream KV snapshot write failures in the storage adapter. + instrument: counter + unit: "{failure}" + stability: development + attributes: + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.append.duration + type: metric + metric_name: decider.append.duration + annotations: + code_generation: + metric_value_type: double + brief: Duration of appending decided events to a JetStream-backed stream. + instrument: histogram + unit: "s" + stability: development + attributes: + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.append.conflicts + type: metric + metric_name: decider.append.conflicts + annotations: + code_generation: + metric_value_type: int + brief: Optimistic concurrency conflicts (wrong expected stream version) surfaced by an append. + instrument: counter + unit: "{conflict}" + stability: development + attributes: + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.replay.duration + type: metric + metric_name: decider.replay.duration + annotations: + code_generation: + metric_value_type: double + brief: Duration of replaying events from an ordered, subject-filtered JetStream consumer. + instrument: histogram + unit: "s" + stability: development + attributes: + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.replay.retries + type: metric + metric_name: decider.replay.retries + annotations: + code_generation: + metric_value_type: int + brief: Ordered-consumer replay attempts retried after a transient JetStream failure. + instrument: counter + unit: "{attempt}" + stability: development + attributes: + - ref: stream_id + requirement_level: recommended + + - id: metric.decider.wasm.execution.duration + type: metric + metric_name: decider.wasm.execution.duration + annotations: + code_generation: + metric_value_type: double + brief: Duration of a WASM guest execution phase. + instrument: histogram + unit: "s" + stability: development + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: required + + - id: metric.decider.wasm.fuel.consumed + type: metric + metric_name: decider.wasm.fuel.consumed + annotations: + code_generation: + metric_value_type: int + brief: Wasmtime fuel consumed by a WASM guest execution phase. + instrument: histogram + unit: "{fuel_unit}" + stability: development + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: required + + - id: metric.decider.wasm.traps + type: metric + metric_name: decider.wasm.traps + annotations: + code_generation: + metric_value_type: int + brief: WASM guest call traps, by classification. + instrument: counter + unit: "{trap}" + stability: development + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: recommended + - ref: trap_classification + requirement_level: required + + - id: span.decider.execute_command + type: span + span_kind: internal + stability: development + brief: Execution of a single decider command, from snapshot and stream read through decide and append. + attributes: + - ref: stream_id + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: write_precondition + requirement_level: recommended + - ref: decision_outcome + requirement_level: + conditionally_required: When execution completes without an error. + + - id: span.decider.read_snapshot + type: span + span_kind: internal + stability: development + brief: Reading a decider snapshot before replaying a command. + attributes: + - ref: stream_id + requirement_level: recommended + - ref: snapshot_outcome + requirement_level: + conditionally_required: When the read completes without an error. + + - id: span.decider.write_snapshot + type: span + span_kind: internal + stability: development + brief: Writing a decider snapshot after a command decision. + attributes: + - ref: stream_id + requirement_level: recommended + - ref: snapshot_write_success + requirement_level: + conditionally_required: When the write attempt completes. + + - id: span.decider.append_stream + type: span + span_kind: client + stability: development + brief: Appending decided events to a JetStream-backed stream. + attributes: + - ref: stream_id + requirement_level: recommended + - ref: write_precondition + requirement_level: recommended + + - id: span.decider.replay_stream + type: span + span_kind: client + stability: development + brief: Replaying events from an ordered, subject-filtered JetStream consumer. + attributes: + - ref: stream_id + requirement_level: recommended + + - id: span.decider.wasm.instantiate + type: span + span_kind: internal + stability: development + brief: Instantiating a WASM decider guest component for a command. + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: required + - ref: trap_classification + requirement_level: + conditionally_required: When the guest call traps. + + - id: span.decider.wasm.replay + type: span + span_kind: internal + stability: development + brief: Replaying prior events into a WASM decider guest session. + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: required + - ref: trap_classification + requirement_level: + conditionally_required: When the guest call traps. + + - id: span.decider.wasm.decide + type: span + span_kind: internal + stability: development + brief: Calling a WASM decider guest's decide export. + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: required + - ref: decision_outcome + requirement_level: + conditionally_required: When the call completes without a trap. + - ref: trap_classification + requirement_level: + conditionally_required: When the guest call traps. + + - id: span.decider.wasm.snapshot + type: span + span_kind: internal + stability: development + brief: Reading or writing a WASM decider guest session's snapshot. + attributes: + - ref: module_name + requirement_level: recommended + - ref: module_version + requirement_level: recommended + - ref: command_type + requirement_level: recommended + - ref: guest_phase + requirement_level: required diff --git a/rsworkspace/Cargo.lock b/rsworkspace/Cargo.lock index 0eb308028..6b44347c2 100644 --- a/rsworkspace/Cargo.lock +++ b/rsworkspace/Cargo.lock @@ -6472,13 +6472,18 @@ dependencies = [ "bytes", "chrono", "futures", + "opentelemetry", "serde", "serde_json", + "testcontainers-modules", "thiserror 2.0.18", "time", "tokio", + "tokio-util", + "tracing", "trogon-decider-runtime", "trogon-nats", + "trogon-semconv", "trogon-std", "uuid", ] @@ -6489,12 +6494,14 @@ version = "0.1.0" dependencies = [ "chrono", "futures", + "opentelemetry", "serde", "serde_json", "thiserror 2.0.18", "tokio", "tracing", "trogon-decider", + "trogon-semconv", "trogon-std", "uuid", ] @@ -6507,7 +6514,13 @@ dependencies = [ "buffa-types", "tempfile", "thiserror 2.0.18", + "tracing", + "trogon-decider", + "trogon-decider-guest-sdk", + "trogon-decider-test", + "trogon-decider-wasm-runtime", "trogon-decider-wit", + "trogon-scheduler-domain", "trogon-std", "trogonai-proto", "wasmparser 0.252.0", @@ -6520,16 +6533,14 @@ name = "trogon-decider-test" version = "0.1.0" dependencies = [ "anyhow", + "base64", "buffa", - "buffa-types", "clap", "serde", "serde_json", "serde_yaml", "thiserror 2.0.18", - "trogon-decider", "trogon-decider-sim", - "trogon-decider-wit", "trogonai-proto", ] @@ -6540,11 +6551,13 @@ dependencies = [ "buffa", "buffa-types", "chrono", + "opentelemetry", "thiserror 2.0.18", "tokio", "tracing", "trogon-decider-runtime", "trogon-decider-wit", + "trogon-semconv", "trogon-std", "trogonai-proto", "uuid", diff --git a/rsworkspace/Cargo.toml b/rsworkspace/Cargo.toml index 5537e8837..7d5053c32 100644 --- a/rsworkspace/Cargo.toml +++ b/rsworkspace/Cargo.toml @@ -137,6 +137,7 @@ reqwest = { version = "=0.12.28", default-features = false, features = ["json", rrule = "=0.14.0" sha2 = "=0.11.0" wiremock = "=0.6.5" +testcontainers-modules = { version = "=0.15.0", features = ["nats"] } tempfile = "=3.27.0" filetime = "=0.2.29" time = "=0.3.53" diff --git a/rsworkspace/cli/trogon-decider-test/Cargo.toml b/rsworkspace/cli/trogon-decider-test/Cargo.toml index 01eedc943..88dc86eb9 100644 --- a/rsworkspace/cli/trogon-decider-test/Cargo.toml +++ b/rsworkspace/cli/trogon-decider-test/Cargo.toml @@ -8,20 +8,22 @@ publish = false [lints] workspace = true +[lib] +name = "trogon_decider_test" +path = "src/lib.rs" + [[bin]] name = "decider-test" path = "src/main.rs" [dependencies] anyhow = { workspace = true } +base64 = { workspace = true } buffa = { workspace = true } -buffa-types = { workspace = true } clap = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } thiserror = { workspace = true } -trogon-decider = { workspace = true } trogon-decider-sim = { workspace = true } -trogon-decider-wit = { workspace = true, features = ["host"] } trogonai-proto = { path = "../../crates/trogonai-proto", default-features = false, features = ["schedules"] } diff --git a/rsworkspace/cli/trogon-decider-test/schedules.yaml b/rsworkspace/cli/trogon-decider-test/schedules.yaml index 6f8994580..3db63786c 100644 --- a/rsworkspace/cli/trogon-decider-test/schedules.yaml +++ b/rsworkspace/cli/trogon-decider-test/schedules.yaml @@ -1,4 +1,9 @@ suite: scheduler.schedules +events: + - trogonai.scheduler.schedules.v1.ScheduleCreated + - trogonai.scheduler.schedules.v1.SchedulePaused + - trogonai.scheduler.schedules.v1.ScheduleResumed + - trogonai.scheduler.schedules.v1.ScheduleRemoved scenarios: - name: create schedule from initial state given: [] @@ -9,13 +14,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{"kind":"heartbeat"}' + data: { utf8: '{"kind":"heartbeat"}' } then: events: - '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.ScheduleCreated @@ -24,13 +30,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{"kind":"heartbeat"}' + data: { utf8: '{"kind":"heartbeat"}' } - name: reject create when schedule already exists given: @@ -40,13 +47,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{"kind":"heartbeat"}' + data: { utf8: '{"kind":"heartbeat"}' } when: '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule schedule_id: backup @@ -54,13 +62,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{"kind":"heartbeat"}' + data: { utf8: '{"kind":"heartbeat"}' } then: rejected: true @@ -72,13 +81,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{}' + data: { utf8: '{}' } when: '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.PauseSchedule schedule_id: backup @@ -103,13 +113,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{}' + data: { utf8: '{}' } - '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.SchedulePaused schedule_id: backup when: @@ -128,13 +139,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{}' + data: { utf8: '{}' } when: '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.RemoveSchedule schedule_id: backup @@ -151,13 +163,14 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{}' + data: { utf8: '{}' } - '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.ScheduleRemoved schedule_id: backup when: @@ -167,12 +180,61 @@ scenarios: scheduled: {} schedule: every: - seconds: 30 + every: 30s delivery: - subject: agent.run + nats_message: + subject: agent.run message: content: content_type: application/json - data: '{"kind":"heartbeat"}' + data: { utf8: '{"kind":"heartbeat"}' } then: rejected: true + + - name: create, pause, then resume a schedule across one session + steps: + - when: + '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule + schedule_id: backup + status: + scheduled: {} + schedule: + every: + every: 30s + delivery: + nats_message: + subject: agent.run + message: + content: + content_type: application/json + data: { utf8: '{"kind":"heartbeat"}' } + then: + events: + - '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.ScheduleCreated + schedule_id: backup + status: + scheduled: {} + schedule: + every: + every: 30s + delivery: + nats_message: + subject: agent.run + message: + content: + content_type: application/json + data: { utf8: '{"kind":"heartbeat"}' } + - when: + '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.PauseSchedule + schedule_id: backup + then: + events: + - '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.SchedulePaused + schedule_id: backup + - when: + '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.ResumeSchedule + schedule_id: backup + then: + events: + - '@type': type.googleapis.com/trogonai.scheduler.schedules.v1.ScheduleResumed + schedule_id: backup diff --git a/rsworkspace/cli/trogon-decider-test/src/codec.rs b/rsworkspace/cli/trogon-decider-test/src/codec.rs index 991f1ef80..8214a6874 100644 --- a/rsworkspace/cli/trogon-decider-test/src/codec.rs +++ b/rsworkspace/cli/trogon-decider-test/src/codec.rs @@ -1,59 +1,32 @@ -//! Typed JSON ↔ proto encoding for decider-test YAML suites. - -use anyhow::{Context, Result, bail}; -use buffa::{Message as _, MessageField, MessageName as _}; -use trogon_decider_wit::host::{self, CommandEnvelope}; -use trogonai_proto::content::v1alpha1 as content_v1alpha1; -use trogonai_proto::scheduler::schedules::{ - CREATE_SCHEDULE_TYPE_URL, PAUSE_SCHEDULE_TYPE_URL, REMOVE_SCHEDULE_TYPE_URL, RESUME_SCHEDULE_TYPE_URL, - v1 as schedules_v1, -}; - -pub fn json_any_to_command(value: &serde_json::Value) -> Result { - let type_url = any_type_url(value)?; - match type_url.as_str() { - CREATE_SCHEDULE_TYPE_URL => Ok(CommandEnvelope { - type_: type_url, - payload: parse_create_schedule_command(value)?.encode_to_vec(), - }), - PAUSE_SCHEDULE_TYPE_URL => Ok(CommandEnvelope { - type_: type_url, - payload: parse_schedule_id_command::(value)?.encode_to_vec(), - }), - REMOVE_SCHEDULE_TYPE_URL => Ok(CommandEnvelope { - type_: type_url, - payload: parse_schedule_id_command::(value)?.encode_to_vec(), - }), - RESUME_SCHEDULE_TYPE_URL => Ok(CommandEnvelope { - type_: type_url, - payload: parse_schedule_id_command::(value)?.encode_to_vec(), - }), - other => bail!("unsupported command type '{other}'"), - } -} - -pub fn json_any_to_envelope(value: &serde_json::Value) -> Result { - let type_url = any_type_url(value)?; - let message_name = type_url_to_message_name(&type_url); - match message_name.as_str() { - schedules_v1::ScheduleCreated::FULL_NAME => Ok(host::AnyEnvelope { - type_: schedules_v1::ScheduleCreated::FULL_NAME.to_string(), - payload: parse_schedule_created(value)?.encode_to_vec(), - }), - schedules_v1::SchedulePaused::FULL_NAME => Ok(host::AnyEnvelope { - type_: schedules_v1::SchedulePaused::FULL_NAME.to_string(), - payload: parse_schedule_paused(value)?.encode_to_vec(), - }), - schedules_v1::ScheduleResumed::FULL_NAME => Ok(host::AnyEnvelope { - type_: schedules_v1::ScheduleResumed::FULL_NAME.to_string(), - payload: parse_schedule_resumed(value)?.encode_to_vec(), - }), - schedules_v1::ScheduleRemoved::FULL_NAME => Ok(host::AnyEnvelope { - type_: schedules_v1::ScheduleRemoved::FULL_NAME.to_string(), - payload: parse_schedule_removed(value)?.encode_to_vec(), - }), - other => bail!("unsupported event type '{other}'"), - } +//! Generic JSON ↔ proto wire encoding for decider-test YAML suites. +//! +//! Conversion is driven entirely by `buffa`'s compiled-in type registry: a +//! decider's proto package registers its message types once, via its +//! generated `register_types` function, and every `@type`-tagged YAML value +//! is then encoded to (or decoded from) wire bytes generically by looking up +//! that type's JSON codec in the registry. A new decider under test needs a +//! YAML suite plus one `register_types` call added to [`registry`], not +//! hand-written per-field parsing code. +//! +//! Canonical proto JSON requires `bytes` fields as base64 strings; so suites +//! stay human-readable, a `bytes` value may instead be written as the wrapper +//! object `{ utf8: "..." }`, which is expanded to the base64 encoding of the +//! string's UTF-8 bytes before the registry codec runs. + +use std::sync::OnceLock; + +use anyhow::{Context, Result}; +use base64::Engine as _; +use buffa::type_registry::TypeRegistry; +use trogon_decider_sim::WireEnvelope; + +fn registry() -> &'static TypeRegistry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(|| { + let mut registry = TypeRegistry::new(); + trogonai_proto::scheduler::schedules::v1::register_types(&mut registry); + registry + }) } pub fn any_type_url(value: &serde_json::Value) -> Result { @@ -72,179 +45,64 @@ pub fn normalize_type_url(type_url: &str) -> String { } } -fn type_url_to_message_name(type_url: &str) -> String { - type_url - .trim_start_matches("type.googleapis.com/") - .trim_start_matches('/') - .to_string() -} - -fn parse_create_schedule_command(value: &serde_json::Value) -> Result { - Ok(schedules_v1::CreateSchedule { - schedule_id: required_string(value, "schedule_id")?, - status: MessageField::some(parse_schedule_status(value.get("status"))?), - schedule: MessageField::some(parse_schedule(value.get("schedule"))?), - delivery: MessageField::some(parse_delivery(value.get("delivery"))?), - message: MessageField::some(parse_message(value.get("message"))?), - }) -} - -fn parse_schedule_id_command

(value: &serde_json::Value) -> Result

-where - P: ScheduleIdCommand, -{ - P::from_schedule_id(required_string(value, "schedule_id")?) -} - -trait ScheduleIdCommand: buffa::Message { - fn from_schedule_id(schedule_id: String) -> Result; -} - -impl ScheduleIdCommand for schedules_v1::PauseSchedule { - fn from_schedule_id(schedule_id: String) -> Result { - Ok(Self { schedule_id }) - } -} - -impl ScheduleIdCommand for schedules_v1::RemoveSchedule { - fn from_schedule_id(schedule_id: String) -> Result { - Ok(Self { schedule_id }) - } -} - -impl ScheduleIdCommand for schedules_v1::ResumeSchedule { - fn from_schedule_id(schedule_id: String) -> Result { - Ok(Self { schedule_id }) - } -} - -fn parse_schedule_created(value: &serde_json::Value) -> Result { - Ok(schedules_v1::ScheduleCreated { - schedule_id: required_string(value, "schedule_id")?, - status: MessageField::some(parse_schedule_status(value.get("status"))?), - schedule: MessageField::some(parse_schedule(value.get("schedule"))?), - delivery: MessageField::some(parse_delivery(value.get("delivery"))?), - message: MessageField::some(parse_message(value.get("message"))?), - }) -} - -fn parse_schedule_paused(value: &serde_json::Value) -> Result { - Ok(schedules_v1::SchedulePaused { - schedule_id: required_string(value, "schedule_id")?, - }) -} - -fn parse_schedule_resumed(value: &serde_json::Value) -> Result { - Ok(schedules_v1::ScheduleResumed { - schedule_id: required_string(value, "schedule_id")?, - }) -} - -fn parse_schedule_removed(value: &serde_json::Value) -> Result { - Ok(schedules_v1::ScheduleRemoved { - schedule_id: required_string(value, "schedule_id")?, - }) -} - -fn parse_schedule_status(value: Option<&serde_json::Value>) -> Result { - let value = value.context("missing schedule status")?; - if value.get("scheduled").is_some() { - return Ok(schedules_v1::ScheduleStatus { - kind: Some(schedules_v1::schedule_status::Scheduled {}.into()), - }); - } - if value.get("paused").is_some() { - return Ok(schedules_v1::ScheduleStatus { - kind: Some(schedules_v1::schedule_status::Paused {}.into()), - }); - } - bail!("schedule status must contain scheduled or paused") -} - -fn parse_schedule(value: Option<&serde_json::Value>) -> Result { - let value = value.context("missing schedule")?; - if let Some(every) = value.get("every") { - let seconds = every - .get("seconds") - .and_then(serde_json::Value::as_i64) - .context("every schedule missing seconds")?; - let nanos = every.get("nanos").and_then(serde_json::Value::as_i64).unwrap_or(0); - if !(0..=999_999_999).contains(&nanos) { - bail!("every.nanos must be between 0 and 999999999"); +/// Rewrites `{ utf8: "..." }` wrapper objects into the base64 string canonical +/// proto JSON requires for `bytes` fields, so suite authors can keep payloads +/// human-readable instead of hand-encoding them. +fn expand_utf8_wrappers(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(map) => { + if map.len() == 1 + && let Some(serde_json::Value::String(text)) = map.get("utf8") + { + let encoded = base64::engine::general_purpose::STANDARD.encode(text.as_bytes()); + *value = serde_json::Value::String(encoded); + return; + } + for child in map.values_mut() { + expand_utf8_wrappers(child); + } } - return Ok(schedules_v1::Schedule { - kind: Some( - schedules_v1::schedule::Every { - every: MessageField::some(buffa_types::google::protobuf::Duration { - seconds, - nanos: nanos as i32, - ..buffa_types::google::protobuf::Duration::default() - }), - } - .into(), - ), - }); - } - bail!("unsupported schedule shape in test YAML") -} - -fn parse_delivery(value: Option<&serde_json::Value>) -> Result { - let value = value.context("missing delivery")?; - let nats_message = value.get("nats_message"); - if value.get("ttl").is_some() - || value.get("source").is_some() - || nats_message.and_then(|inner| inner.get("ttl")).is_some() - || nats_message.and_then(|inner| inner.get("source")).is_some() - { - bail!("delivery.ttl/source are not supported yet in test YAML"); - } - let subject = value - .get("subject") - .or_else(|| nats_message.and_then(|inner| inner.get("subject"))) - .and_then(serde_json::Value::as_str) - .context("delivery missing subject")?; - Ok(schedules_v1::Delivery { - kind: Some( - schedules_v1::delivery::NatsMessage { - subject: subject.to_string(), - ttl: MessageField::none(), - source: MessageField::none(), + serde_json::Value::Array(items) => { + for item in items { + expand_utf8_wrappers(item); } - .into(), - ), - }) -} - -fn parse_message(value: Option<&serde_json::Value>) -> Result { - let value = value.context("missing message")?; - if value.get("headers").is_some() { - bail!("message.headers are not supported yet in test YAML"); + } + _ => {} } - let content = value.get("content").context("message missing content")?; - let data = content - .get("data") - .and_then(serde_json::Value::as_str) - .context("message content missing data")?; - Ok(schedules_v1::Message { - content: MessageField::some(content_v1alpha1::Content { - content_type: content - .get("content_type") - .or_else(|| content.get("contentType")) - .and_then(serde_json::Value::as_str) - .unwrap_or("application/json") - .to_string(), - data: data.as_bytes().to_vec(), - }), - headers: Vec::new(), - }) } -fn required_string(value: &serde_json::Value, field: &str) -> Result { - value - .get(field) - .and_then(serde_json::Value::as_str) - .map(str::to_string) - .with_context(|| format!("missing {field}")) +/// Encodes an `@type`-tagged YAML value to wire bytes via the type's +/// registered JSON codec, returning the normalized type URL alongside the +/// payload. +fn json_any_to_wire(value: &serde_json::Value) -> Result<(String, Vec)> { + let type_url = any_type_url(value)?; + let entry = registry() + .json_any_by_url(&type_url) + .with_context(|| format!("unregistered type '{type_url}'"))?; + let mut fields = value.clone(); + if let serde_json::Value::Object(map) = &mut fields { + map.remove("@type"); + } + expand_utf8_wrappers(&mut fields); + let payload = + (entry.from_json)(fields).map_err(|message| anyhow::anyhow!("failed to encode '{type_url}': {message}"))?; + Ok((type_url, payload)) +} + +/// Decodes a scenario `when` value into the command envelope the guest +/// expects, tagged with the type's full `type.googleapis.com/` URL. +pub fn json_any_to_command(value: &serde_json::Value) -> Result { + let (type_url, payload) = json_any_to_wire(value)?; + Ok(WireEnvelope::new(type_url, payload)) +} + +/// Decodes a scenario `given`/`then.events` value into the event envelope +/// the guest expects, tagged with the type's bare protobuf full name (what +/// the guest itself emits, unlike the URL-prefixed command envelope type). +pub fn json_any_to_envelope(value: &serde_json::Value) -> Result { + let (type_url, payload) = json_any_to_wire(value)?; + let type_url = type_url.trim_start_matches("type.googleapis.com/").to_string(); + Ok(WireEnvelope::new(type_url, payload)) } #[cfg(test)] diff --git a/rsworkspace/cli/trogon-decider-test/src/codec/tests.rs b/rsworkspace/cli/trogon-decider-test/src/codec/tests.rs index 3387b5a8f..83c7ad1dd 100644 --- a/rsworkspace/cli/trogon-decider-test/src/codec/tests.rs +++ b/rsworkspace/cli/trogon-decider-test/src/codec/tests.rs @@ -1,3 +1,6 @@ +use buffa::Message as _; +use trogonai_proto::scheduler::schedules::v1 as schedules_v1; + use super::*; #[test] @@ -8,30 +11,73 @@ fn normalize_adds_prefix() { ); } -fn error_message(result: Result) -> String { - match result { - Ok(_) => String::new(), - Err(error) => error.to_string(), - } +fn create_schedule_value() -> serde_json::Value { + serde_json::json!({ + "@type": "type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule", + "schedule_id": "backup", + "status": { "scheduled": {} }, + "schedule": { "every": { "every": "30s" } }, + "delivery": { "nats_message": { "subject": "agent.run" } }, + "message": { "content": { "content_type": "application/json", "data": "e30=" } }, + }) } #[test] -fn every_rejects_out_of_range_nanos() { - let value = serde_json::json!({ "every": { "seconds": 1, "nanos": 1_000_000_000_i64 } }); - let error = error_message(parse_schedule(Some(&value))); - assert!(error.contains("nanos"), "unexpected error: {error:?}"); +fn json_any_to_command_encodes_via_registry() { + let command = json_any_to_command(&create_schedule_value()).expect("encode command"); + assert_eq!( + command.type_url, + "type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule" + ); + + let decoded = schedules_v1::CreateSchedule::decode_from_slice(&command.payload).expect("decode command"); + assert_eq!(decoded.schedule_id, "backup"); + + let round_tripped_text = + trogonai_proto::decode_event_to_json("trogonai.scheduler.schedules.v1.CreateSchedule", &command.payload) + .expect("registered type") + .expect("decode to json"); + let round_tripped: serde_json::Value = serde_json::from_str(&round_tripped_text).expect("valid json"); + assert_eq!(round_tripped["delivery"]["natsMessage"]["subject"], "agent.run"); +} + +#[test] +fn json_any_to_envelope_uses_bare_full_name() { + let value = serde_json::json!({ + "@type": "trogonai.scheduler.schedules.v1.SchedulePaused", + "schedule_id": "backup", + }); + let envelope = json_any_to_envelope(&value).expect("encode envelope"); + assert_eq!(envelope.type_url, "trogonai.scheduler.schedules.v1.SchedulePaused"); + + let decoded = schedules_v1::SchedulePaused::decode_from_slice(&envelope.payload).expect("decode envelope"); + assert_eq!(decoded.schedule_id, "backup"); +} + +#[test] +fn utf8_wrapper_encodes_identically_to_base64() { + let mut wrapped = create_schedule_value(); + wrapped["message"]["content"]["data"] = serde_json::json!({ "utf8": "{}" }); + + let from_wrapper = json_any_to_command(&wrapped).expect("encode wrapped command"); + let from_base64 = json_any_to_command(&create_schedule_value()).expect("encode base64 command"); + assert_eq!(from_wrapper.payload, from_base64.payload); + + let decoded = schedules_v1::CreateSchedule::decode_from_slice(&from_wrapper.payload).expect("decode command"); + let content = decoded.message.expect("message").content.expect("content"); + assert_eq!(content.data, b"{}"); } #[test] -fn delivery_rejects_unsupported_ttl() { - let value = serde_json::json!({ "subject": "agent.run", "ttl": "60s" }); - let error = error_message(parse_delivery(Some(&value))); - assert!(error.contains("ttl"), "unexpected error: {error:?}"); +fn missing_type_is_an_error() { + let value = serde_json::json!({ "schedule_id": "backup" }); + let error = json_any_to_command(&value).unwrap_err().to_string(); + assert!(error.contains("@type"), "unexpected error: {error}"); } #[test] -fn message_rejects_unsupported_headers() { - let value = serde_json::json!({ "headers": [], "content": { "data": "{}" } }); - let error = error_message(parse_message(Some(&value))); - assert!(error.contains("headers"), "unexpected error: {error:?}"); +fn unregistered_type_is_an_error() { + let value = serde_json::json!({ "@type": "type.googleapis.com/trogonai.scheduler.schedules.v1.NoSuchType" }); + let error = json_any_to_command(&value).unwrap_err().to_string(); + assert!(error.contains("unregistered"), "unexpected error: {error}"); } diff --git a/rsworkspace/cli/trogon-decider-test/src/lib.rs b/rsworkspace/cli/trogon-decider-test/src/lib.rs new file mode 100644 index 000000000..57c8cfe4f --- /dev/null +++ b/rsworkspace/cli/trogon-decider-test/src/lib.rs @@ -0,0 +1,13 @@ +//! Parsing for `decider-test` YAML suites, shared by the `decider-test` binary and by +//! `trogon-decider-sim`'s parity integration test. +//! +//! Keeping `Suite`/`Scenario`/`Step`/`Then` parsing (and their conversion into +//! [`trogon_decider_sim::ScenarioIr`]) in a library target means a test that wants to run every +//! scenario in a real YAML suite through both a native decider and a compiled wasm component can +//! reuse this exact parser, rather than re-implementing YAML-to-IR conversion a second time. +#![cfg_attr(test, allow(clippy::expect_used, clippy::panic, clippy::unwrap_used))] + +pub mod codec; +mod suite; + +pub use suite::{ErrorExpectation, Scenario, Step, Suite, Then}; diff --git a/rsworkspace/cli/trogon-decider-test/src/main.rs b/rsworkspace/cli/trogon-decider-test/src/main.rs index eb7de3ccf..2bc1a01fc 100644 --- a/rsworkspace/cli/trogon-decider-test/src/main.rs +++ b/rsworkspace/cli/trogon-decider-test/src/main.rs @@ -1,4 +1,4 @@ -mod codec; +#![cfg_attr(test, allow(clippy::expect_used, clippy::panic, clippy::unwrap_used))] use std::collections::BTreeSet; use std::fs; @@ -7,10 +7,9 @@ use std::process; use anyhow::{Context, Result, bail}; use clap::Parser; -use serde::Deserialize; -use trogon_decider_sim::{SimHost, SimScenario}; - -use crate::codec::{any_type_url, json_any_to_command, json_any_to_envelope}; +use trogon_decider_sim::{SimHost, SimInstance}; +use trogon_decider_test::codec::{any_type_url, normalize_type_url}; +use trogon_decider_test::{Scenario, Suite, Then}; #[derive(Parser)] #[command( @@ -22,6 +21,12 @@ struct Args { #[arg(long, default_value = "human")] format: String, + /// Downgrade zero-coverage declared commands/events from a failure to a + /// warning. By default a declared command or event with no coverage + /// across every scenario fails the run. + #[arg(long)] + no_strict: bool, + /// Compiled decider component wasm: PathBuf, @@ -36,56 +41,6 @@ enum OutputFormat { Tap, } -#[derive(Debug, Deserialize)] -struct Suite { - suite: String, - scenarios: Vec, -} - -#[derive(Debug, Deserialize)] -struct Scenario { - name: String, - #[serde(default)] - given: Vec, - when: serde_json::Value, - then: Then, -} - -#[derive(Debug, Deserialize)] -#[serde(untagged)] -#[allow(dead_code)] -enum Then { - Events { events: Vec }, - Error { error: ErrorExpectation }, - Rejected { rejected: bool }, -} - -/// `then.error` accepts either a bare string or the documented `{ code, message }` -/// object; both are matched against the domain error's code or message. -#[derive(Debug, Deserialize)] -#[serde(untagged)] -enum ErrorExpectation { - Structured { - #[serde(default)] - code: Option, - #[serde(default)] - message: Option, - }, - Plain(String), -} - -impl ErrorExpectation { - fn expected(&self) -> Result { - match self { - Self::Plain(value) => Ok(value.clone()), - Self::Structured { code, message } => code - .clone() - .or_else(|| message.clone()) - .context("then.error requires a code or message"), - } - } -} - fn main() { if let Err(error) = run() { eprintln!("error: {error:#}"); @@ -97,26 +52,43 @@ fn run() -> Result<()> { let args = Args::parse(); let output_format = parse_output_format(&args.format)?; let wasm_bytes = fs::read(&args.wasm).with_context(|| format!("read {}", args.wasm.display()))?; - let suite: Suite = serde_yaml::from_str( - &fs::read_to_string(&args.suite).with_context(|| format!("read {}", args.suite.display()))?, - )?; + let suite = + Suite::from_yaml(&fs::read_to_string(&args.suite).with_context(|| format!("read {}", args.suite.display()))?)?; let host = SimHost::load(&wasm_bytes)?; - let declared = host + let declared_commands = host .instantiate(())? .descriptor()? .commands .into_iter() .map(|spec| spec.command_type) .collect::>(); - let mut exercised = BTreeSet::new(); + let declared_events = suite + .events + .iter() + .map(|type_url| normalize_type_url(type_url)) + .collect::>(); + let mut exercised_commands = BTreeSet::new(); + let mut exercised_events = BTreeSet::new(); let mut failures = 0usize; + for scenario in &suite.scenarios { - // Fresh component per scenario so guest-global state cannot leak between runs. let mut instance = host.instantiate(())?; - let when_type = any_type_url(&scenario.when)?; - exercised.insert(when_type.clone()); + let steps = scenario.steps()?; + + for value in &scenario.given { + exercised_events.insert(any_type_url(value)?); + } + for (when, then) in &steps { + exercised_commands.insert(any_type_url(when)?); + if let Then::Events { events } = then { + for value in events { + exercised_events.insert(any_type_url(value)?); + } + } + } + match run_scenario(&mut instance, scenario) { Ok(()) => { if matches!(output_format, OutputFormat::Tap) { @@ -136,8 +108,10 @@ fn run() -> Result<()> { } } - for command_type in declared.difference(&exercised) { - eprintln!("warning: declared command never exercised as when: {command_type}"); + let command_gaps = report_coverage_gaps(&declared_commands, &exercised_commands, "command", args.no_strict); + let event_gaps = report_coverage_gaps(&declared_events, &exercised_events, "event", args.no_strict); + if !args.no_strict && (command_gaps > 0 || event_gaps > 0) { + bail!("{command_gaps} declared command(s) and {event_gaps} declared event(s) have zero scenario coverage"); } if failures > 0 { @@ -146,40 +120,28 @@ fn run() -> Result<()> { Ok(()) } -fn run_scenario(instance: &mut trogon_decider_sim::SimInstance<()>, scenario: &Scenario) -> Result<()> { - let given = scenario - .given - .iter() - .map(json_any_to_envelope) - .collect::>>()?; - let when = json_any_to_command(&scenario.when)?; - - match &scenario.then { - Then::Events { events } => { - let expected = events.iter().map(json_any_to_envelope).collect::>>()?; - SimScenario::new() - .given(given) - .when(when) - .then_events(expected) - .run(instance) - .map_err(anyhow::Error::new) - } - Then::Rejected { rejected } => { - let scenario = SimScenario::new().given(given).when(when); - let scenario = if *rejected { - scenario.then_rejected() - } else { - scenario.then_accepted() - }; - scenario.run(instance).map_err(anyhow::Error::new) - } - Then::Error { error } => SimScenario::new() - .given(given) - .when(when) - .then_error(error.expected()?) - .run(instance) - .map_err(anyhow::Error::new), +/// Reports every `declared` type with zero coverage in `exercised`, at +/// `warning` level under `--no-strict` and `error` level otherwise, and +/// returns how many gaps were found. The caller decides whether a nonzero +/// strict-mode count fails the run, so both the command and event checks +/// always run and report in full before the run bails. +fn report_coverage_gaps( + declared: &BTreeSet, + exercised: &BTreeSet, + kind: &str, + no_strict: bool, +) -> usize { + let gaps: Vec<&String> = declared.difference(exercised).collect(); + let level = if no_strict { "warning" } else { "error" }; + for gap in &gaps { + eprintln!("{level}: declared {kind} never exercised in any scenario: {gap}"); } + gaps.len() +} + +fn run_scenario(instance: &mut SimInstance<()>, scenario: &Scenario) -> Result<()> { + let ir = scenario.to_ir()?; + ir.to_sim_scenario().run(instance).map_err(anyhow::Error::new) } fn parse_output_format(raw: &str) -> Result { diff --git a/rsworkspace/cli/trogon-decider-test/src/suite.rs b/rsworkspace/cli/trogon-decider-test/src/suite.rs new file mode 100644 index 000000000..a58fb1444 --- /dev/null +++ b/rsworkspace/cli/trogon-decider-test/src/suite.rs @@ -0,0 +1,177 @@ +//! YAML test suite shapes, and their conversion into [`trogon_decider_sim::ScenarioIr`]. +//! +//! Parsing (this module) and execution ([`crate::codec`] plus `trogon_decider_sim`) are kept +//! separate from the coverage-gap bookkeeping that only the `decider-test` binary needs, so a +//! parity test elsewhere in the workspace can parse a real YAML suite and run every scenario +//! through both a native decider and a compiled wasm component using this exact code path, +//! rather than a second hand-written parser. + +use anyhow::{Context, Result, bail}; +use serde::Deserialize; +use trogon_decider_sim::{ExpectedOutcome, ScenarioIr, ScenarioStep}; + +use crate::codec; + +/// A YAML decider conformance suite: a named group of scenarios plus the events the suite +/// author declares the decider under test can produce. +#[derive(Debug, Deserialize)] +pub struct Suite { + /// The suite's human-readable name. + pub suite: String, + /// Type URLs (bare or `type.googleapis.com/`-prefixed) of every event the decider under + /// test can produce, asserted by the suite author. Compared against every event actually + /// referenced across all scenarios' `given` and `then.events` entries for the strict + /// coverage check, since neither the WIT descriptor nor the proto type registry can + /// distinguish "events this decider emits" from unrelated message and helper types. + #[serde(default)] + pub events: Vec, + /// The suite's scenarios, run independently against a fresh component instance. + pub scenarios: Vec, +} + +impl Suite { + /// Parses a suite from its YAML text. + pub fn from_yaml(yaml: &str) -> Result { + serde_yaml::from_str(yaml).context("parse suite YAML") + } + + /// Converts every scenario in this suite into [`ScenarioIr`], in declared order. + pub fn to_ir(&self) -> Result> { + self.scenarios.iter().map(Scenario::to_ir).collect() + } +} + +/// One scenario: an optional seeded history, plus one or more `when`/`then` steps. +#[derive(Debug, Deserialize)] +pub struct Scenario { + /// The scenario's human-readable name. + pub name: String, + /// The seeded event history, replayed before the first step's command is decided. + #[serde(default)] + pub given: Vec, + /// The legacy single-step command, mutually exclusive with `steps`. + #[serde(default)] + pub when: Option, + /// The legacy single-step expectation, mutually exclusive with `steps`. + #[serde(default)] + pub then: Option, + /// An ordered `when`/`then` sequence run against a single open session, each step's emitted + /// events folded in before the next step's command is decided. Mutually exclusive with the + /// single `when`/`then` shape. + #[serde(default)] + pub steps: Option>, +} + +/// One `when`/`then` pair in a scenario's `steps` list. +#[derive(Debug, Deserialize)] +pub struct Step { + /// The command decided against the state accumulated so far. + pub when: serde_json::Value, + /// The outcome this step's command is expected to produce. + pub then: Then, +} + +impl Scenario { + /// Returns this scenario's ordered when/then steps, normalizing the legacy single + /// `when`/`then` shape into a one-element step list. + /// + /// Exactly one of `steps` (nonempty) or `when`+`then` must be present; any other + /// combination is a malformed scenario. + pub fn steps(&self) -> Result> { + match (self.steps.as_ref(), self.when.as_ref(), self.then.as_ref()) { + (Some(steps), None, None) => { + if steps.is_empty() { + bail!("scenario '{}' has an empty steps list", self.name); + } + Ok(steps.iter().map(|step| (&step.when, &step.then)).collect()) + } + (None, Some(when), Some(then)) => Ok(vec![(when, then)]), + _ => bail!( + "scenario '{}' must have exactly one of: a nonempty steps list, or both when and then", + self.name + ), + } + } + + /// Converts this scenario into decider-agnostic [`ScenarioIr`], decoding every `given`, + /// `when`, and `then.events` value into wire form via [`crate::codec`]. + pub fn to_ir(&self) -> Result { + let mut ir = ScenarioIr::new(self.name.clone()); + for value in &self.given { + ir.given.push(codec::json_any_to_envelope(value)?); + } + for (when, then) in self.steps()? { + let when = codec::json_any_to_command(when)?; + let expect = match then { + Then::Events { events } => { + let events = events + .iter() + .map(codec::json_any_to_envelope) + .collect::>>()?; + ExpectedOutcome::Events(events) + } + Then::Rejected { rejected: true } => ExpectedOutcome::Rejected, + Then::Rejected { rejected: false } => ExpectedOutcome::Accepted, + Then::Error { error } => ExpectedOutcome::Error(error.expected()?), + }; + ir.steps.push(ScenarioStep { when, expect }); + } + Ok(ir) + } +} + +/// A step's declared expectation. +#[derive(Debug, Deserialize)] +#[serde(untagged)] +#[allow(dead_code)] +pub enum Then { + /// The command must be accepted and produce exactly these events, in order. + Events { + /// The expected events, in order. + events: Vec, + }, + /// The command must fail with this code or message. + Error { + /// The expected error. + error: ErrorExpectation, + }, + /// The command must be accepted (`true`) or rejected (`false`), without asserting events. + Rejected { + /// Whether the command must be rejected. + rejected: bool, + }, +} + +/// `then.error` accepts either a bare string or the documented `{ code, message }` object; both +/// are matched against the domain error's code or message. +#[derive(Debug, Deserialize)] +#[serde(untagged)] +pub enum ErrorExpectation { + /// The documented `{ code, message }` shape. + Structured { + /// The expected domain error code. + #[serde(default)] + code: Option, + /// The expected domain error message. + #[serde(default)] + message: Option, + }, + /// A bare string, matched against either the domain error's code or message. + Plain(String), +} + +impl ErrorExpectation { + /// Returns the string this expectation is matched against. + pub fn expected(&self) -> Result { + match self { + Self::Plain(value) => Ok(value.clone()), + Self::Structured { code, message } => code + .clone() + .or_else(|| message.clone()) + .context("then.error requires a code or message"), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/cli/trogon-decider-test/src/suite/tests.rs b/rsworkspace/cli/trogon-decider-test/src/suite/tests.rs new file mode 100644 index 000000000..a96ba7346 --- /dev/null +++ b/rsworkspace/cli/trogon-decider-test/src/suite/tests.rs @@ -0,0 +1,150 @@ +use super::*; + +fn error_expectation(yaml: &str) -> String { + match serde_yaml::from_str::(yaml) { + Ok(Then::Error { error }) => error.expected().unwrap_or_default(), + _ => String::new(), + } +} + +#[test] +fn then_error_accepts_structured_code() { + assert_eq!(error_expectation("error:\n code: already-exists\n"), "already-exists"); +} + +#[test] +fn then_error_accepts_plain_string() { + assert_eq!(error_expectation("error: already-exists\n"), "already-exists"); +} + +fn scenario(when: Option, then: Option, steps: Option>) -> Scenario { + Scenario { + name: "scenario under test".to_string(), + given: Vec::new(), + when, + then, + steps, + } +} + +#[test] +fn steps_normalizes_legacy_when_then_shape() { + let scenario = scenario( + Some(serde_json::json!({"@type": "a"})), + Some(Then::Rejected { rejected: true }), + None, + ); + assert_eq!(scenario.steps().expect("valid scenario").len(), 1); +} + +#[test] +fn steps_returns_ordered_step_list() { + let scenario = scenario( + None, + None, + Some(vec![ + Step { + when: serde_json::json!({"@type": "a"}), + then: Then::Rejected { rejected: true }, + }, + Step { + when: serde_json::json!({"@type": "b"}), + then: Then::Rejected { rejected: false }, + }, + ]), + ); + assert_eq!(scenario.steps().expect("valid scenario").len(), 2); +} + +#[test] +fn steps_rejects_empty_steps_list() { + let scenario = scenario(None, None, Some(Vec::new())); + let error = scenario.steps().unwrap_err().to_string(); + assert!(error.contains("empty steps list"), "unexpected error: {error}"); +} + +#[test] +fn steps_rejects_mixing_steps_and_when_then() { + let scenario = scenario( + Some(serde_json::json!({"@type": "a"})), + Some(Then::Rejected { rejected: true }), + Some(vec![Step { + when: serde_json::json!({"@type": "a"}), + then: Then::Rejected { rejected: true }, + }]), + ); + let error = scenario.steps().unwrap_err().to_string(); + assert!(error.contains("exactly one of"), "unexpected error: {error}"); +} + +#[test] +fn steps_rejects_missing_when_and_then() { + let scenario = scenario(None, None, None); + let error = scenario.steps().unwrap_err().to_string(); + assert!(error.contains("exactly one of"), "unexpected error: {error}"); +} + +#[test] +fn to_ir_converts_given_steps_and_events() { + let scenario = Scenario { + name: "create schedule".to_string(), + given: Vec::new(), + when: None, + then: None, + steps: Some(vec![Step { + when: serde_json::json!({ + "@type": "type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule", + "schedule_id": "backup", + "status": { "scheduled": {} }, + "schedule": { "every": { "every": "30s" } }, + "delivery": { "nats_message": { "subject": "agent.run" } }, + "message": { "content": { "content_type": "application/json", "data": "e30=" } }, + }), + then: Then::Events { + events: vec![serde_json::json!({ + "@type": "trogonai.scheduler.schedules.v1.ScheduleCreated", + "schedule_id": "backup", + "status": { "scheduled": {} }, + "schedule": { "every": { "every": "30s" } }, + "delivery": { "nats_message": { "subject": "agent.run" } }, + "message": { "content": { "content_type": "application/json", "data": "e30=" } }, + })], + }, + }]), + }; + + let ir = scenario.to_ir().expect("scenario converts to ir"); + assert_eq!(ir.name, "create schedule"); + assert!(ir.given.is_empty()); + assert_eq!(ir.steps.len(), 1); + assert_eq!( + ir.steps[0].when.type_url, + "type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule" + ); + assert!(matches!(&ir.steps[0].expect, ExpectedOutcome::Events(events) if events.len() == 1)); +} + +#[test] +fn to_ir_maps_rejected_and_accepted() { + let rejected = scenario( + Some(serde_json::json!({ + "@type": "type.googleapis.com/trogonai.scheduler.schedules.v1.PauseSchedule", + "schedule_id": "backup", + })), + Some(Then::Rejected { rejected: true }), + None, + ); + let ir = rejected.to_ir().expect("scenario converts to ir"); + assert!(matches!(ir.steps[0].expect, ExpectedOutcome::Rejected)); + + let accepted = scenario( + Some(serde_json::json!({ + "@type": "type.googleapis.com/trogonai.scheduler.schedules.v1.PauseSchedule", + "schedule_id": "backup", + })), + Some(Then::Rejected { rejected: false }), + None, + ); + let ir = accepted.to_ir().expect("scenario converts to ir"); + assert!(matches!(ir.steps[0].expect, ExpectedOutcome::Accepted)); +} diff --git a/rsworkspace/cli/trogon-decider-test/src/tests.rs b/rsworkspace/cli/trogon-decider-test/src/tests.rs index 2079c85a7..ad82a4439 100644 --- a/rsworkspace/cli/trogon-decider-test/src/tests.rs +++ b/rsworkspace/cli/trogon-decider-test/src/tests.rs @@ -1,18 +1,26 @@ use super::*; -fn error_expectation(yaml: &str) -> String { - match serde_yaml::from_str::(yaml) { - Ok(Then::Error { error }) => error.expected().unwrap_or_default(), - _ => String::new(), - } +fn set(values: &[&str]) -> BTreeSet { + values.iter().map(|value| value.to_string()).collect() } #[test] -fn then_error_accepts_structured_code() { - assert_eq!(error_expectation("error:\n code: already-exists\n"), "already-exists"); +fn coverage_gaps_are_counted() { + let declared = set(&["a", "b"]); + let exercised = set(&["a"]); + assert_eq!(report_coverage_gaps(&declared, &exercised, "command", false), 1); } #[test] -fn then_error_accepts_plain_string() { - assert_eq!(error_expectation("error: already-exists\n"), "already-exists"); +fn coverage_gaps_zero_when_fully_covered() { + let declared = set(&["a"]); + let exercised = set(&["a"]); + assert_eq!(report_coverage_gaps(&declared, &exercised, "command", false), 0); +} + +#[test] +fn coverage_gaps_counted_regardless_of_strict_flag() { + let declared = set(&["a", "b"]); + let exercised = set(&["a"]); + assert_eq!(report_coverage_gaps(&declared, &exercised, "command", true), 1); } diff --git a/rsworkspace/crates/trogon-decider-guest-macros/Cargo.toml b/rsworkspace/crates/trogon-decider-guest-macros/Cargo.toml index 91d11bc48..2684b8db5 100644 --- a/rsworkspace/crates/trogon-decider-guest-macros/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-guest-macros/Cargo.toml @@ -8,20 +8,28 @@ publish = false [lib] proc-macro = true -[lints] -workspace = true +[lints.rust] +missing_docs = "warn" +warnings = "deny" +unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(dylint_lib, values(any()))'] } + +[lints.clippy] +all = "deny" +expect_used = "deny" +panic = "deny" +unwrap_used = "deny" [dependencies] proc-macro2 = "1.0.106" quote = "1.0.45" syn = { version = "2.0.117", features = ["full"] } +trogon-decider-wit = { workspace = true } [dev-dependencies] trybuild = "1" thiserror = { workspace = true } trogon-decider = { workspace = true } trogon-decider-guest-sdk = { workspace = true } -trogon-decider-wit = { workspace = true, features = ["guest"] } trogon-scheduler-domain = { path = "../trogon-scheduler-domain" } trogonai-proto = { path = "../trogonai-proto", default-features = false, features = ["schedules"] } wit-bindgen = { workspace = true } diff --git a/rsworkspace/crates/trogon-decider-guest-macros/README.md b/rsworkspace/crates/trogon-decider-guest-macros/README.md new file mode 100644 index 000000000..ead509573 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-guest-macros/README.md @@ -0,0 +1,20 @@ +# trogon-decider-guest-macros + +The `-macros` suffix (per [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md)) marks a +proc-macro crate; this one owns exactly one function-like macro, `export_decider!`, which turns +one or more `Decider` implementations sharing a `State`/`Event` into a compiled WASM +component's WIT glue: a hidden `wit_bindgen::generate!` bindings module, a `Component` unit +struct implementing the WIT `Guest` trait, and a `Session` struct implementing `GuestSession` +(`new`, `evolve`, `decide`, `snapshot`) around the native decider's own `evaluate_decision` +logic. It reads `trogon_decider_wit::WIT_DIR` at this crate's own compile time so the generated +`wit_bindgen::generate!` call's `path:` stays an absolute path, independent of the calling +component crate's directory depth. + +This crate is a macro-expansion detail; application code depends on it indirectly through +`trogon-decider-guest-sdk`, which re-exports `export_decider!` alongside the trait/codec glue +the macro's generated code calls into. + +See [`trogon-decider-guest-sdk`'s README](../trogon-decider-guest-sdk/README.md) for the +`export_decider!` call shape, and +[Decider Platform](../../../docs/architecture/decider.md) for how the generated component fits +into the WASM execution path. diff --git a/rsworkspace/crates/trogon-decider-guest-macros/src/lib.rs b/rsworkspace/crates/trogon-decider-guest-macros/src/lib.rs index 1731056ca..85a1a138b 100644 --- a/rsworkspace/crates/trogon-decider-guest-macros/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-guest-macros/src/lib.rs @@ -19,6 +19,27 @@ pub fn export_decider(input: TokenStream) -> TokenStream { .into(); } + // `trogon_decider_wit::WIT_DIR` is this crate's own dependency on `trogon-decider-wit`, + // resolved (and baked into a `&'static str`) when *this* proc-macro crate was compiled. + // Reading it here, rather than emitting `env!("CARGO_MANIFEST_DIR")` into the generated + // tokens, keeps `wit_bindgen::generate!`'s `path:` absolute and independent of the + // directory depth of whichever crate calls `export_decider!`. + let wit_dir = trogon_decider_wit::WIT_DIR; + if !std::path::Path::new(wit_dir).is_dir() { + return syn::Error::new( + proc_macro2::Span::call_site(), + format!( + "export_decider!: WIT contract directory not found at '{wit_dir}' \ + (resolved from trogon_decider_wit::WIT_DIR); this points at the \ + trogon-decider-wit crate that trogon-decider-guest-macros itself \ + depends on, so a missing directory means that crate's checkout is \ + incomplete or its `wit/` directory was moved", + ), + ) + .to_compile_error() + .into(); + } + let canonical = &commands[0]; let canonical_ty = &canonical.ty; let module_name = &canonical.module; @@ -204,7 +225,7 @@ pub fn export_decider(input: TokenStream) -> TokenStream { mod __trogon_decider_bindings { wit_bindgen::generate!({ world: "decider", - path: "../../crates/trogon-decider-wit/wit", + path: #wit_dir, generate_all, }); @@ -222,6 +243,7 @@ pub fn export_decider(input: TokenStream) -> TokenStream { Self { code: value.code, message: value.message, + details: value.details, } } } @@ -300,6 +322,7 @@ pub fn export_decider(input: TokenStream) -> TokenStream { other => Err(__trogon_decider_bindings::DomainError { code: "invalid-command".to_string(), message: format!("unknown command type '{other}'"), + details: Vec::new(), }), } } @@ -348,6 +371,7 @@ pub fn export_decider(input: TokenStream) -> TokenStream { __trogon_decider_bindings::DomainError { code: "invalid-command".to_string(), message: format!("unknown command type '{other}'"), + details: Vec::new(), }, )), } diff --git a/rsworkspace/crates/trogon-decider-guest-macros/tests/compile_fail.rs b/rsworkspace/crates/trogon-decider-guest-macros/tests/compile_fail.rs index 944d9c8dd..c46cfd2c3 100644 --- a/rsworkspace/crates/trogon-decider-guest-macros/tests/compile_fail.rs +++ b/rsworkspace/crates/trogon-decider-guest-macros/tests/compile_fail.rs @@ -1,60 +1,8 @@ -#![allow(clippy::expect_used)] - -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; +//! Trybuild harness asserting the `tests/ui/*.rs` fixtures fail to compile with the expected +//! diagnostics. #[test] fn ui() { - // `export_decider!` embeds `wit_bindgen::generate!({ path: "../../crates/trogon-decider-wit/wit" })`, - // resolved relative to the crate being compiled. trybuild compiles each UI test from a - // generated crate under `/tests/trybuild/`, where that relative path would not - // resolve — the WIT-read failure would then mask the very compile errors these tests - // assert (e.g. the bundle shared-event bound). Provision the contract WIT at the - // location trybuild's generated crates resolve against so the macro expands fully. - provision_trybuild_wit(); - let t = trybuild::TestCases::new(); t.compile_fail("tests/ui/*.rs"); } - -fn provision_trybuild_wit() { - let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); - let wit_src = manifest.join("../trogon-decider-wit/wit/world.wit"); - // trybuild stages each generated crate under `/tests/trybuild//`, where the - // macro's `wit_bindgen::generate!({ path: "../../crates/trogon-decider-wit/wit" })` resolves - // to `/tests/crates/trogon-decider-wit/wit`. Derive `` from Cargo rather than - // assuming the default tree so a custom CARGO_TARGET_DIR stages the WIT where trybuild looks. - let dst_dir = cargo_target_dir().join("tests/crates/trogon-decider-wit/wit"); - fs::create_dir_all(&dst_dir).expect("create trybuild WIT dir"); - fs::copy(&wit_src, dst_dir.join("world.wit")).expect("copy world.wit into trybuild tree"); -} - -/// Resolve Cargo's target directory the same way trybuild does, so the staged WIT lands under the -/// root trybuild actually uses (it queries `cargo metadata`, which already honors CARGO_TARGET_DIR -/// and `build.target-dir`). Falls back to the default workspace tree only if the query fails. -fn cargo_target_dir() -> PathBuf { - let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); - let metadata = Command::new(env!("CARGO")) - .args(["metadata", "--format-version", "1", "--no-deps"]) - .current_dir(manifest) - .output(); - - if let Ok(output) = metadata - && output.status.success() - && let Some(dir) = parse_target_directory(&output.stdout) - { - return dir; - } - - manifest.join("../../target") -} - -fn parse_target_directory(metadata: &[u8]) -> Option { - let text = std::str::from_utf8(metadata).ok()?; - let key = "\"target_directory\":\""; - let start = text.find(key)? + key.len(); - let rest = &text[start..]; - let end = rest.find('"')?; - Some(PathBuf::from(&rest[..end])) -} diff --git a/rsworkspace/crates/trogon-decider-guest-sdk/Cargo.toml b/rsworkspace/crates/trogon-decider-guest-sdk/Cargo.toml index 4335de3d8..06c175384 100644 --- a/rsworkspace/crates/trogon-decider-guest-sdk/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-guest-sdk/Cargo.toml @@ -9,8 +9,16 @@ publish = false default = [] test-support = [] -[lints] -workspace = true +[lints.rust] +missing_docs = "warn" +warnings = "deny" +unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(dylint_lib, values(any()))'] } + +[lints.clippy] +all = "deny" +expect_used = "deny" +panic = "deny" +unwrap_used = "deny" [dependencies] buffa = { workspace = true } diff --git a/rsworkspace/crates/trogon-decider-guest-sdk/README.md b/rsworkspace/crates/trogon-decider-guest-sdk/README.md new file mode 100644 index 000000000..b643c0ee0 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-guest-sdk/README.md @@ -0,0 +1,196 @@ +# trogon-decider-guest-sdk + +Bridges a native [`Decider`](../trogon-decider) implementation to the Trogon WIT guest +contract, so it can be compiled to a WASM component and executed by +[`trogon-decider-wasm-runtime`](../trogon-decider-wasm-runtime) (or any other host that binds the +same `trogon:decider` world). + +This guide covers the `Decider` trait, the `export_decider!` macro, the WIT contract it targets, +building a component, and the two ways to test one. For the end-to-end picture, including how +this crate's WIT contract and engine budgets fit into the rest of the platform, see +[Decider Platform](../../../docs/architecture/decider.md). + +## The `Decider` trait + +A decider is a plain Rust type implementing `trogon_decider::Decider`: + +```rust +pub trait Decider: Sized { + type StreamId: ?Sized; + type State; + type Event; + type DecideError: std::error::Error; + type EvolveError: std::error::Error; + + const WRITE_PRECONDITION: Option = None; + + fn stream_id(&self) -> &Self::StreamId; + fn initial_state() -> Self::State; + fn evolve(state: Self::State, event: &Self::Event) -> Result; + fn decide(state: &Self::State, command: &Self) -> Result, Self::DecideError>; +} +``` + +`decide` returns a `Decision`, which is either a flat batch of events +(`Decision::Events`) or an `Act` plan: a chain of steps, each re-evaluating `decide` against the +state produced by the previous step, that folds into one ordered event batch. The guest bridge +(`decide_command` below) evaluates both forms through the same `evaluate_decision` entry point +the native runtime uses, so a decider's WASM behavior is identical to its native behavior, +`Decision::Act` included. + +`Self::Event` must also implement `EventEncode` (`encode(&self) -> Result, Error>`), +`EventType` (`event_type(&self) -> Result<&'static str, Error>`), and `EventDecode` +(`decode(EventData<'_>) -> Result, Error>`, where `EventDecodeOutcome` +is `Decoded(event)` or `Skipped` for envelopes outside this decider's event set) so the bridge +can serialize decided events and replay stored ones. + +## `export_decider!` + +`export_decider!` generates the WIT component glue (`wit_bindgen::generate!`, the exported +`Guest`/`GuestSession` impls, snapshot load/store, command dispatch) for one or more commands +that share a `Decider::State` and `Decider::Event`: + +```rust +use trogon_decider_guest_sdk::export_decider; + +export_decider!( + CreateSchedule { + type_url = CREATE_SCHEDULE_TYPE_URL, + proto = v1::CreateSchedule, + module = "scheduler.schedules", + version = "0.1.0", + state_schema_version = SCHEDULES_STATE_SCHEMA_VERSION, + write_precondition = no_stream, + }, + PauseSchedule { + type_url = PAUSE_SCHEDULE_TYPE_URL, + proto = v1::PauseSchedule, + module = "scheduler.schedules", + version = "0.1.0", + state_schema_version = SCHEDULES_STATE_SCHEMA_VERSION, + }, +); +``` + +Each entry is `CommandType { ... }`, where `CommandType` implements `Decider` (and `TryFrom` +for the wire command). Fields: + +- `type_url`: the protobuf `Any` type URL this command decodes from. +- `proto`: the wire (buffa-generated) protobuf message type; `CommandType: TryFrom`. +- `module` / `version`: the exported `ModuleDescriptor`'s name and version. Every command in a + bundle must declare the same values (checked at macro-expansion time). +- `state_schema_version`: fed to the snapshot codec; every command in a bundle must declare the + same value (also checked at macro-expansion time, since a bundle shares one `State`). +- `write_precondition` (optional): `any`, `stream_exists`, `no_stream`, or `default` (the + decider's own `Decider::WRITE_PRECONDITION`, which is also the default when omitted). + +When a bundle has more than one command, `export_decider!` requires every command's `Decider` +impl to share the canonical (first-listed) command's `State` and `Event` types, and asserts this +independently of the WIT bindings so a mismatch fails to compile with a clear error rather than a +confusing `wit-bindgen` diagnostic. + +The macro resolves its WIT contract via `trogon_decider_wit::WIT_DIR`, an absolute path baked +into `trogon-decider-wit` at that crate's own compile time. This means a component crate can live +at any directory depth relative to `trogon-decider-wit`; the macro does not depend on the calling +crate's location. + +## WIT contract summary + +The contract lives in `trogon-decider-wit`'s `wit/world.wit` (package `trogon:decider`, world +`decider`, interface `handler`): + +- `module-descriptor { name, version, commands: list }` and + `command-spec { command-type, write-precondition: option }`, returned by + `descriptor()`. +- `command-envelope { type, payload }` and `any-envelope { type, payload }`: type-URL-tagged + protobuf bytes for commands and events, respectively. +- `domain-error { code, message, details: list> }`: a stable machine + `code`, a human-readable `message`, and `details` as ordered key/value pairs carrying the + error's `#[source]` chain (`("cause.0", ...)`, `("cause.1", ...)`, ...) below `message`, so + causes that `message`'s single `Display` line would otherwise erase remain available in + structured form. +- `decide-error` is a variant of `rejected(domain-error)` or `faulted(domain-error)`. +- `resource session { constructor(snapshot), evolve(events), decide(command), snapshot() }`: one + session per component instance, holding in-memory state across a `decide`/`evolve` cycle. + +## Building a component + +A component crate needs: + +```toml +[lib] +crate-type = ["cdylib"] + +[dependencies] +trogon-decider = { workspace = true } +trogon-decider-guest-sdk = { workspace = true } +wit-bindgen = { workspace = true } +# ...your domain crate(s), proto crate(s) + +[package.metadata.component] +package = "trogon:decider" +``` + +`wit-bindgen` must be a direct dependency of the component crate itself (not just of +`trogon-decider-guest-macros`), because `export_decider!`'s generated code calls +`wit_bindgen::generate!` unqualified, resolving in the caller's own dependency graph. + +Place the crate under `rsworkspace/wasm-components//` and build it with: + +```sh +mise run artifacts:wasm-components +``` + +which builds every crate under `wasm-components/*` with `cargo component build --target +wasm32-unknown-unknown --release` and copies each `.wasm` (and its `.binpb` FileDescriptorSet, if +one was generated) to `dist/decider/.wasm`, where `` is the crate name with the +`trogon-` prefix and `-decider` suffix stripped. You can also build a single component directly: + +```sh +cargo component build -p --target wasm32-unknown-unknown --release +``` + +## Testing + +Two complementary paths exist, both driving the built `.wasm` through an in-memory wasmtime host +rather than a real deployment: + +### `trogon-decider-sim` + +`trogon-decider-sim` is a Rust-level in-memory host (`SimHost`, `SimInstance`, `SimScenario`) for +integration tests that construct protobuf messages directly in Rust. Add a fixture accessor next +to `SimFixture::schedules()` in `trogon-decider-sim/src/fixture.rs` (feature `test-support`) +pointing at your component's built `.wasm` path, then write a test under +`trogon-decider-sim/tests/` following the pattern in `tests/schedules.rs`: build a `CommandEnvelope` +protobuf, run it through `SimScenario`, and assert on the resulting `AnyEnvelope`s or the returned +`decide-error`. + +### `decider-test` CLI + +`cli/trogon-decider-test` runs a YAML conformance suite against a compiled component without +writing any Rust: + +```sh +cargo build -p trogon-decider-test --release +./target/release/decider-test dist/decider/.wasm cli/trogon-decider-test/.yaml +``` + +A suite file has a top-level `suite` name and a list of `scenarios`, each with: + +- `given`: a list of protobuf-JSON `Any` events (`'@type'` plus the message's fields) to replay + into state before the command runs. +- `when`: the protobuf-JSON `Any` command to decide. +- `then`: one of + - `events: [...]`: the exact ordered list of protobuf-JSON `Any` events the command must + produce. + - `rejected: true`: the command must be rejected (`decide-error`), without checking `code` or + `message`. + - `error: { code: ..., message: ... }` (or a bare string): the command must be rejected or + faulted with a matching `code` or `message`. + +Proto `bytes` fields take canonical-JSON base64 strings, or the human-readable wrapper +`{ utf8: '...' }`, which the runner expands to the base64 encoding of the string's UTF-8 bytes +(e.g. `data: { utf8: '{"kind":"heartbeat"}' }`). + +See `cli/trogon-decider-test/schedules.yaml` for a full example covering creation, rejection, and +multi-step scenarios. diff --git a/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge.rs b/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge.rs index 14962d394..c8baa96d3 100644 --- a/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge.rs +++ b/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge.rs @@ -1,25 +1,52 @@ -use trogon_decider::{Decider, Decision, EventData, EventDecode, EventDecodeOutcome, EventEncode, EventType}; +use std::error::Error as _; +use trogon_decider::{ + Decider, DecisionFailure, EventData, EventDecode, EventDecodeOutcome, EventEncode, EventType, evaluate_decision, +}; + +/// Abstraction over whatever concrete WIT-generated command-envelope type a guest's generated +/// bindings produce, so the bridge functions in this module stay generic over the exact struct +/// shape. pub trait CommandEnvelopeView { + /// The envelope's persistent command-type string, used to dispatch to the right decoder. fn command_type(&self) -> &str; + /// The raw, still-encoded command payload bytes. fn command_payload(&self) -> &[u8]; } +/// Abstraction over whatever concrete WIT-generated stored-event envelope type a guest's +/// generated bindings produce, mirroring the WIT `any-envelope` record (`%type`/`payload`). pub trait AnyEnvelopeView { + /// The stored event's persistent event-type string, used to dispatch to the right decoder. fn event_type(&self) -> &str; + /// The raw, still-encoded event payload bytes. fn event_payload(&self) -> &[u8]; } -/// Wire projection of a domain error: the `code`/`message` strings the WIT `domain-error` +/// Wire projection of a domain error: the `code`/`message`/`details` the WIT `domain-error` /// record carries. Produced from a typed [`BridgeError`] at the boundary, never built from a /// stringified source mid-pipeline. +/// +/// `details` carries the error's `#[source]` chain below `message` as ordered `("cause.N", +/// text)` pairs, so causes `message`'s single `Display` line would otherwise erase crossing the +/// WIT boundary remain available in structured form. pub struct DomainErrorParts { + /// Stable, machine-readable rejection code (see [`BridgeError::code`]). pub code: String, + /// The error's rendered `Display` text. pub message: String, + /// The error's `#[source]` chain below `message`, as ordered `("cause.N", text)` pairs (see + /// [`causal_chain`]). + pub details: Vec<(String, String)>, } +/// Constructs a caller's own decide-error type from [`DomainErrorParts`], routing business-rule +/// rejections and processing failures to two different constructors (see [`into_decide_error`] +/// for how [`BridgeError::is_rejection`] selects between them). pub trait DecideErrorView { + /// Builds the caller's error type for a business-rule rejection raised by `decide`. fn rejected(parts: DomainErrorParts) -> Self; + /// Builds the caller's error type for any other processing failure in the bridge pipeline. fn faulted(parts: DomainErrorParts) -> Self; } @@ -30,28 +57,38 @@ pub trait DecideErrorView { /// `Display` only when projecting into [`DomainErrorParts`] at the WIT boundary. #[derive(Debug, thiserror::Error)] pub enum BridgeError { + /// The incoming command envelope's type URL did not match this decider's expected command + /// type. #[error("unknown command type '{actual}'")] - UnknownCommandType { actual: String }, + UnknownCommandType { + /// The type URL actually carried by the envelope. + actual: String, + }, + /// The command envelope's payload failed to decode as the expected protobuf/`buffa` + /// message. #[error("failed to decode command payload: {0}")] CommandDecode(#[source] Box), + /// The decoded protobuf command failed `TryFrom` conversion into the decider's typed + /// command. #[error("failed to convert command: {0}")] CommandConvert(#[source] Box), + /// A stored event envelope's payload failed to decode during replay. #[error("failed to decode event: {0}")] EventDecode(#[source] Box), + /// `Decider::evolve` failed while folding a decoded event into state. #[error("failed to evolve state: {0}")] Evolve(#[source] Box), + /// A business-rule rejection raised by `decide`. #[error("{source}")] Rejected { - code: String, + /// The original error returned by `decide`. #[source] source: Box, }, - #[error("Act decisions are not supported in WASM v1")] - UnsupportedAct, - #[error("unsupported decision variant")] - UnsupportedDecision, + /// An event produced by `decide` failed to encode to its wire payload. #[error("failed to encode event: {0}")] EventEncode(#[source] Box), + /// An event produced by `decide` failed to resolve its persistent event-type string. #[error("failed to resolve event type: {0}")] EventTypeResolve(#[source] Box), } @@ -62,8 +99,7 @@ impl BridgeError { Self::UnknownCommandType { .. } | Self::CommandDecode(_) | Self::CommandConvert(_) => "invalid-command", Self::EventDecode(_) => "decode-failed", Self::Evolve(_) => "evolve-failed", - Self::Rejected { code, .. } => code, - Self::UnsupportedAct | Self::UnsupportedDecision => "unsupported-decision", + Self::Rejected { .. } => "rejected", Self::EventEncode(_) | Self::EventTypeResolve(_) => "encode-failed", } } @@ -75,13 +111,28 @@ impl BridgeError { impl From for DomainErrorParts { fn from(error: BridgeError) -> Self { + let details = causal_chain(error.source()); Self { code: error.code().to_string(), message: error.to_string(), + details, } } } +/// Flattens a `#[source]` chain into ordered `("cause.N", text)` pairs, starting one level +/// below the error whose `Display` already produced [`DomainErrorParts::message`]. +fn causal_chain(mut source: Option<&(dyn std::error::Error + 'static)>) -> Vec<(String, String)> { + let mut details = Vec::new(); + let mut depth = 0usize; + while let Some(error) = source { + details.push((format!("cause.{depth}"), error.to_string())); + source = error.source(); + depth += 1; + } + details +} + /// Project a [`BridgeError`] into a caller-facing error built from [`DomainErrorParts`]. fn into_view>(error: BridgeError) -> E { DomainErrorParts::from(error).into() @@ -97,11 +148,19 @@ fn into_decide_error(error: BridgeError) -> D { } } +/// Wire-agnostic bag of an event's type name and encoded payload bytes, the encoded-event +/// counterpart to [`DomainErrorParts`]. A caller's own event-envelope type is built +/// `From`. pub struct AnyEnvelopeParts { + /// The event's persistent type name. pub type_url: String, + /// The event's encoded wire payload. pub payload: Vec, } +/// Decodes and converts an incoming command envelope into the caller's typed command `Cmd`, +/// checking the envelope's type URL first and returning the caller's error type `E` (built via +/// [`DomainErrorParts`]) on any failure. pub fn decode_command(expected_type_url: &str, envelope: Env) -> Result where Env: CommandEnvelopeView, @@ -121,6 +180,10 @@ where Cmd::try_from(proto).map_err(|source| into_view(BridgeError::CommandConvert(Box::new(source)))) } +/// Decodes one stored event envelope and folds it into decider state via `C::evolve`. Envelopes +/// whose event type isn't recognized by this decider's event set are skipped (not errored), +/// mirroring the native runtime's replay behavior so WASM and native replay fold the same stream +/// to the same state. #[allow( clippy::disallowed_methods, reason = "decider guest bridge dispatch path; the disallowed_methods rule targets test code calling decide/evolve directly" @@ -146,32 +209,42 @@ where C::evolve(state, &event).map_err(|source| into_view(BridgeError::Evolve(Box::new(source)))) } +/// Project a [`DecisionFailure`] into a [`BridgeError`], mirroring the native runtime's +/// handling of the same failure type (see `trogon-decider-runtime`'s `append_decision` and +/// `trogon_decider::testing`'s `decide_events`). +fn map_decision_failure(failure: DecisionFailure) -> BridgeError +where + C: Decider, +{ + match failure { + DecisionFailure::Decide(source) => BridgeError::Rejected { + source: Box::new(source), + }, + DecisionFailure::Evolve(source) => BridgeError::Evolve(Box::new(source)), + } +} + +/// Decides and evolves a command through the same [`evaluate_decision`] path the native +/// runtime uses, so a [`Decision::Act`](trogon_decider::Decision::Act) plan's steps observe +/// exactly the state they would observe natively. Only the resulting event batch is encoded +/// here; the caller's `evolve` export folds those events back into session state, matching how +/// the native runtime persists them. #[allow( clippy::disallowed_methods, reason = "decider guest bridge dispatch path; the disallowed_methods rule targets test code calling decide/evolve directly" )] pub fn decide_command(command: &C, state: &C::State) -> Result, D> where - C: Decider, + C: Decider, C::Event: EventEncode + EventType, A: From, D: DecideErrorView, { - let decision = C::decide(state, command).map_err(|source| { - let code = C::decide_error_code(&source).to_string(); - into_decide_error(BridgeError::Rejected { - code, - source: Box::new(source), - }) - })?; - - let events = match decision { - Decision::Events(batch) => batch.into_vec(), - Decision::Act(_) => return Err(into_decide_error(BridgeError::UnsupportedAct)), - _ => return Err(into_decide_error(BridgeError::UnsupportedDecision)), - }; + let (_, events) = evaluate_decision::(state.clone(), command) + .map_err(|failure| into_decide_error(map_decision_failure::(failure)))?; events + .into_vec() .into_iter() .map(encode_event_envelope::) .collect() @@ -194,6 +267,8 @@ where .into()) } +/// Projects the native [`trogon_decider::WritePrecondition`] into the guest-side +/// [`WritePreconditionTag`], preserving `None` (no precondition) as `None`. pub fn map_write_precondition(value: Option) -> Option { value.map(|precondition| match precondition { trogon_decider::WritePrecondition::Any => WritePreconditionTag::Any, @@ -202,8 +277,21 @@ pub fn map_write_precondition(value: Option) }) } +/// Guest-side mirror of [`trogon_decider::WritePrecondition`]. +/// +/// This crate compiles once, generically, before any concrete decider's WIT bindings exist, so +/// it cannot name the WIT-generated `write-precondition` variant that `wit_bindgen::generate!` +/// produces per-crate inside `trogon-decider-guest-macros`. Callers convert this tag into their +/// own generated `WritePrecondition` type (see `map_write_precondition_tag` in +/// `trogon-decider-guest-macros`). pub enum WritePreconditionTag { + /// No constraint on the stream's current state. Any, + /// The stream must already exist. StreamExists, + /// The stream must not yet exist. NoStream, } + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge/tests.rs b/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge/tests.rs new file mode 100644 index 000000000..56ff23daf --- /dev/null +++ b/rsworkspace/crates/trogon-decider-guest-sdk/src/bridge/tests.rs @@ -0,0 +1,249 @@ +use super::*; +use trogon_decider::Decision; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum FixtureEvent { + Opened { id: String }, + Funded { amount: u32 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("fixture codec error")] +struct FixtureCodecError; + +impl EventEncode for FixtureEvent { + type Error = FixtureCodecError; + + fn encode(&self) -> Result, Self::Error> { + match self { + Self::Opened { id } => Ok(id.clone().into_bytes()), + Self::Funded { amount } => Ok(amount.to_le_bytes().to_vec()), + } + } +} + +impl EventType for FixtureEvent { + type Error = std::convert::Infallible; + + fn event_type(&self) -> Result<&'static str, Self::Error> { + Ok(match self { + Self::Opened { .. } => "fixture.opened", + Self::Funded { .. } => "fixture.funded", + }) + } +} + +impl EventDecode for FixtureEvent { + type Error = FixtureCodecError; + + fn decode(event: EventData<'_>) -> Result, Self::Error> { + match event.event_type { + "fixture.opened" => { + let id = String::from_utf8(event.payload.to_vec()).map_err(|_| FixtureCodecError)?; + Ok(EventDecodeOutcome::Decoded(Self::Opened { id })) + } + "fixture.funded" => { + let bytes: [u8; 4] = event.payload.try_into().map_err(|_| FixtureCodecError)?; + Ok(EventDecodeOutcome::Decoded(Self::Funded { + amount: u32::from_le_bytes(bytes), + })) + } + _ => Ok(EventDecodeOutcome::Skipped), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FixtureState { + New, + Opened, + Funded { amount: u32 }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +enum FixtureDecideError { + #[error("account is already open")] + AlreadyOpen, + #[error("account is not open")] + NotOpen, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("invalid fixture event for current state")] +struct FixtureEvolveError; + +/// Fixture command whose `decide` returns a two-step [`Decision::Act`]: the first step opens +/// an account, the second funds it while observing the state the first step's event produced. +struct OpenAndFund { + id: String, + amount: u32, +} + +impl Decider for OpenAndFund { + type StreamId = str; + type State = FixtureState; + type Event = FixtureEvent; + type DecideError = FixtureDecideError; + type EvolveError = FixtureEvolveError; + + fn stream_id(&self) -> &str { + &self.id + } + + fn initial_state() -> FixtureState { + FixtureState::New + } + + fn evolve(state: FixtureState, event: &FixtureEvent) -> Result { + match (state, event) { + (FixtureState::New, FixtureEvent::Opened { .. }) => Ok(FixtureState::Opened), + (FixtureState::Opened, FixtureEvent::Funded { amount }) => Ok(FixtureState::Funded { amount: *amount }), + _ => Err(FixtureEvolveError), + } + } + + fn decide(_state: &FixtureState, _command: &Self) -> Result, FixtureDecideError> { + Decision::act() + .execute(|state: &FixtureState, command: &Self| match state { + FixtureState::New => Ok(Decision::event(FixtureEvent::Opened { id: command.id.clone() })), + _ => Err(FixtureDecideError::AlreadyOpen), + }) + .execute(|state: &FixtureState, command: &Self| match state { + FixtureState::Opened => Ok(Decision::event(FixtureEvent::Funded { amount: command.amount })), + _ => Err(FixtureDecideError::NotOpen), + }) + .into() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct WireEvent { + type_url: String, + payload: Vec, +} + +impl From for WireEvent { + fn from(value: AnyEnvelopeParts) -> Self { + Self { + type_url: value.type_url, + payload: value.payload, + } + } +} + +impl AnyEnvelopeView for WireEvent { + fn event_type(&self) -> &str { + &self.type_url + } + + fn event_payload(&self) -> &[u8] { + &self.payload + } +} + +#[derive(Debug, PartialEq, Eq)] +enum WireErrorKind { + Rejected, + Faulted, +} + +#[derive(Debug, PartialEq, Eq)] +struct WireError { + kind: WireErrorKind, + code: String, +} + +impl DecideErrorView for WireError { + fn rejected(parts: DomainErrorParts) -> Self { + Self { + kind: WireErrorKind::Rejected, + code: parts.code, + } + } + + fn faulted(parts: DomainErrorParts) -> Self { + Self { + kind: WireErrorKind::Faulted, + code: parts.code, + } + } +} + +impl From for WireError { + fn from(parts: DomainErrorParts) -> Self { + Self { + kind: WireErrorKind::Faulted, + code: parts.code, + } + } +} + +fn encode_native_events(events: trogon_decider::Events) -> Vec { + events + .into_vec() + .into_iter() + .map(|event| WireEvent { + type_url: EventType::event_type(&event).expect("infallible").to_string(), + payload: EventEncode::encode(&event).expect("fixture events always encode"), + }) + .collect() +} + +/// The bridge's `decide_command` must produce the exact same ordered event batch as the +/// native `evaluate_decision` path for an `Act`-based decision, since the WIT `decide` export +/// and the native runtime's decision evaluation are expected to be semantically identical. +#[test] +fn act_decision_matches_native_evaluation() { + let state = FixtureState::New; + let command = OpenAndFund { + id: "acct-1".to_string(), + amount: 42, + }; + + let bridge_events = + decide_command::(&command, &state).expect("act decision should succeed"); + + let (native_state, native_events) = + evaluate_decision::(state, &command).expect("act decision should succeed"); + let expected_events = encode_native_events(native_events); + + assert_eq!(bridge_events, expected_events); + assert_eq!(native_state, FixtureState::Funded { amount: 42 }); + + // Folding the bridge's event batch through `evolve_one`, exactly as the host does after + // `decide` returns, must reach the same final state `evaluate_decision` computed natively. + let mut folded_state = FixtureState::New; + for event in bridge_events { + folded_state = evolve_one::(folded_state, event) + .expect("bridge-encoded events must replay cleanly"); + } + assert_eq!(folded_state, native_state); +} + +/// A rejection raised by an `Act` chain's first step, when invoked against a state that step +/// does not accept, must surface through the bridge with the same stable error code the +/// native path would classify it with. +#[test] +fn act_rejection_matches_native_evaluation() { + // An account that is already `Opened` fails the first step's `FixtureState::New` guard, + // proving the bridge's step-level rejection and error code match native evaluation. + let state = FixtureState::Opened; + let command = OpenAndFund { + id: "acct-1".to_string(), + amount: 42, + }; + + let bridge_error = + decide_command::(&command, &state).expect_err("first step should reject"); + assert_eq!(bridge_error.kind, WireErrorKind::Rejected); + assert_eq!(bridge_error.code, "rejected"); + + let native_failure = + evaluate_decision::(state, &command).expect_err("first step should reject natively"); + match native_failure { + trogon_decider::DecisionFailure::Decide(error) => { + assert_eq!(error, FixtureDecideError::AlreadyOpen); + } + trogon_decider::DecisionFailure::Evolve(_) => panic!("expected a decide rejection, not an evolve failure"), + } +} diff --git a/rsworkspace/crates/trogon-decider-guest-sdk/src/lib.rs b/rsworkspace/crates/trogon-decider-guest-sdk/src/lib.rs index 908d76409..367812e75 100644 --- a/rsworkspace/crates/trogon-decider-guest-sdk/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-guest-sdk/src/lib.rs @@ -1,7 +1,8 @@ //! Bridges [`Decider`](trogon_decider::Decider) implementations to the Trogon WIT guest contract. +#![cfg_attr(test, allow(clippy::expect_used, clippy::panic, clippy::unwrap_used))] #![doc = "\n\ -Guest deciders must use [`std::collections::BTreeMap`] (not `HashMap`) for any map fields — \ +Guest deciders must use [`std::collections::BTreeMap`] (not `HashMap`) for any map fields: \ proto `map<>` decoded via buffa seeds a hasher that pulls `wasi:random` on wasip2.\n\ "] diff --git a/rsworkspace/crates/trogon-decider-guest-sdk/src/snapshot.rs b/rsworkspace/crates/trogon-decider-guest-sdk/src/snapshot.rs index 0b57c0ff2..7cb54240a 100644 --- a/rsworkspace/crates/trogon-decider-guest-sdk/src/snapshot.rs +++ b/rsworkspace/crates/trogon-decider-guest-sdk/src/snapshot.rs @@ -4,22 +4,38 @@ /// rather than a string) and only rendered to a message at the WIT-facing boundary. #[derive(Debug, thiserror::Error)] pub enum SnapshotDecodeError { + /// The decoded frame's `schema_version` did not match what the caller expected. #[error("expected schema '{expected}', got '{actual}'")] - SchemaMismatch { expected: String, actual: String }, + SchemaMismatch { + /// The schema version the caller expected to load. + expected: String, + /// The schema version actually stored in the frame. + actual: String, + }, + /// The frame decoded structurally, but the inner [`buffa::Message`] payload failed to + /// decode. #[error("failed to decode snapshot payload: {0}")] Payload(#[source] buffa::DecodeError), + /// The frame's `schema_version` field was not valid UTF-8. #[error("invalid utf8 in schema_version: {0}")] SchemaVersionUtf8(#[source] std::string::FromUtf8Error), + /// The frame had no `schema_version` field (protobuf field 1). #[error("missing schema_version")] MissingSchemaVersion, + /// The frame had no `payload` field (protobuf field 2). #[error("missing payload")] MissingPayload, + /// The byte buffer ended before a field's declared length. #[error("unexpected eof")] UnexpectedEof, + /// A varint exceeded 64 bits, or its final byte carried more than the single valid high bit. #[error("varint overflow")] VarintOverflow, + /// A length-delimited field's declared length would overflow when added to the current + /// offset. #[error("length overflow")] LengthOverflow, + /// An unhandled protobuf wire type was encountered while skipping an unknown field. #[error("unsupported wire type")] UnsupportedWireType, } @@ -57,6 +73,8 @@ where Some(encode_snapshot(state, schema_version)) } +/// Encodes decider state plus a schema version tag into this crate's versioned snapshot frame +/// format. pub fn encode_snapshot(state: &S, schema_version: &str) -> Vec where S: buffa::Message, @@ -64,6 +82,8 @@ where encode_snapshot_frame(schema_version, buffa::Message::encode_to_vec(state)) } +/// Decodes a snapshot frame, validates its `schema_version` matches `expected_schema` (else +/// [`SnapshotDecodeError::SchemaMismatch`]), then decodes the payload as `S`. pub fn decode_snapshot(bytes: &[u8], expected_schema: &str) -> Result where S: buffa::Message, diff --git a/rsworkspace/crates/trogon-decider-nats/Cargo.toml b/rsworkspace/crates/trogon-decider-nats/Cargo.toml index 95ed6c1ac..965e77bc6 100644 --- a/rsworkspace/crates/trogon-decider-nats/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-nats/Cargo.toml @@ -19,19 +19,25 @@ unwrap_used = "deny" [dependencies] thiserror = { workspace = true } -async-nats = { workspace = true, features = ["jetstream", "kv", "server_2_12"] } +async-nats = { workspace = true, features = ["jetstream", "kv", "server_2_10", "server_2_12"] } bytes = { workspace = true } chrono = { version = "0.4", features = ["serde"] } futures = { workspace = true } +opentelemetry = { workspace = true } serde_json = { workspace = true } +tokio = { workspace = true, features = ["macros", "time"] } +tokio-util = { workspace = true } +tracing = { workspace = true } trogon-decider-runtime = { version = "0.1.0", path = "../trogon-decider-runtime" } trogon-nats = { version = "0.1.0", path = "../trogon-nats" } +trogon-semconv = { workspace = true } trogon-std = { version = "0.1.0", path = "../trogon-std", features = ["uuid"] } uuid = { workspace = true } [dev-dependencies] serde = { workspace = true } serde_json = { workspace = true } +testcontainers-modules = { workspace = true } time = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } trogon-nats = { version = "0.1.0", path = "../trogon-nats", features = ["test-support"] } diff --git a/rsworkspace/crates/trogon-decider-nats/README.md b/rsworkspace/crates/trogon-decider-nats/README.md new file mode 100644 index 000000000..91375a355 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/README.md @@ -0,0 +1,20 @@ +# trogon-decider-nats + +NATS JetStream storage adapter for `trogon-decider-runtime`. Owns the mapping from a decider's +logical stream identifiers onto one physical JetStream stream (subject-per-logical-stream, via a +caller-supplied `StreamSubjectResolver`), atomic multi-event append with optimistic concurrency, +ordered subject-filtered replay with bounded retry, and KV-backed snapshot storage +(`snapshot_store`). Per the crate boundaries in [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md), +this crate is the NATS-specific adapter layer; it implements `trogon-decider-runtime`'s storage +traits (`StreamRead`/`StreamAppend`, `SnapshotRead`/`SnapshotWrite`) but knows nothing about +`decide`/`evolve` or command execution itself. + +It also owns two read-side primitives used to build projections and message-driven consumers on +top of decider streams: `Projector` (a generic catch-up driver for subject-filtered JetStream +projections, checkpointed via `ProjectionCheckpointStore`) and `Processor` (a durable pull-consumer +message loop with a `MessageHandler`/`HandlerVerdict`/`RedeliveryPolicy` contract). `provision` +provides idempotent `ensure_stream`/`ensure_bucket` create-or-open helpers used to set up the +underlying JetStream stream and KV bucket before either primitive runs. + +See [docs/architecture/decider.md](../../../docs/architecture/decider.md) for how this adapter fits +into the end-to-end decider platform, including when to reach for `Projector` versus `Processor`. diff --git a/rsworkspace/crates/trogon-decider-nats/src/lib.rs b/rsworkspace/crates/trogon-decider-nats/src/lib.rs index 344c97a7f..aa4a62b34 100644 --- a/rsworkspace/crates/trogon-decider-nats/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-nats/src/lib.rs @@ -10,11 +10,24 @@ mod store; +/// Durable pull-consumer message processing. +pub mod processor; +/// Generic catch-up driver for subject-filtered JetStream projections. +pub mod projector; +/// Idempotent create-or-open provisioning for JetStream streams and buckets. +pub mod provision; /// Snapshot storage helpers backed by JetStream Key/Value. pub mod snapshot_store; /// Event stream storage helpers backed by JetStream streams. pub mod stream_store; +pub use processor::{HandlerVerdict, MessageHandler, PoisonReason, Processor, ProcessorError, RedeliveryPolicy}; +pub use projector::{ + CatchUpError, CatchUpOutcome, CheckpointSequence, ProjectionApply, ProjectionCheckpointStore, Projector, +}; +pub use provision::{ + EnsureBucketError, EnsureStreamError, KvConfigMismatch, StreamConfigMismatch, ensure_bucket, ensure_stream, +}; pub use snapshot_store::{ NatsSnapshotConfig, SnapshotChange, SnapshotCodecError, SnapshotKvError, SnapshotStoreError, checkpoint_key, list_snapshots, maybe_advance_checkpoint, persist_snapshot_change, read_checkpoint, read_snapshot, diff --git a/rsworkspace/crates/trogon-decider-nats/src/processor.rs b/rsworkspace/crates/trogon-decider-nats/src/processor.rs new file mode 100644 index 000000000..2c2ebb2ca --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/processor.rs @@ -0,0 +1,336 @@ +//! Durable pull-consumer message processing. +//! +//! [`Processor`] owns the parts of a durable-consumer worker that stay +//! identical across handlers: provisioning the durable consumer via +//! [`jetstream::stream::Stream::create_consumer_strict`] (create it if +//! absent, fail loudly if an existing consumer's configuration diverges, +//! rather than the looser `get_or_create_consumer` used elsewhere in this +//! workspace, which neither closes the create-then-fetch race nor validates +//! configuration), running the pull loop, and turning a [`MessageHandler`]'s +//! [`HandlerVerdict`] into an ack, a delayed nak, or a term. Redelivery is +//! bounded by [`RedeliveryPolicy`]; shutdown is cooperative via a +//! [`CancellationToken`]. +//! +//! `create_consumer_strict` requires the `server_2_10` `async-nats` feature, +//! which this crate enables. +//! +//! v1 dispatches every message to one handler; there is no per-key lane +//! dispatch (compare the scheduler's dispatcher, which is schedule-key +//! specific across five generic parameters and does not extract cleanly). +//! Callers needing per-key ordering or dispatch should build it on top of +//! [`Processor`], as the scheduler does today. +//! +//! Consumer-stream-level failures (opening the message stream, reading the +//! next message, and settling ack/nak/term) are treated as fatal for +//! [`Processor::run`] and end the loop; only per-message handler outcomes go +//! through the bounded redelivery path. This crate has no logging +//! dependency, so callers that want visibility into a fatal stop should +//! inspect the returned [`ProcessorError`]; [`MessageHandler::on_poison`] is +//! the equivalent hook for per-message terminal outcomes. + +use std::panic::AssertUnwindSafe; +use std::time::Duration; + +use async_nats::jetstream; +use async_nats::jetstream::consumer::pull; +use async_nats::jetstream::consumer::{AckPolicy, PullConsumer}; +use async_nats::jetstream::message::AckKind; +use async_nats::jetstream::stream::{ConsumerCreateStrictError, ConsumerCreateStrictErrorKind}; +use futures::{FutureExt, StreamExt}; +use tokio_util::sync::CancellationToken; + +/// Verdict a [`MessageHandler`] returns for one message. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HandlerVerdict { + /// The message was handled successfully; acknowledge it. + Ack, + /// The message could not be handled right now; redeliver it, subject to + /// [`RedeliveryPolicy`]. + Retry, + /// The message can never be handled; stop redelivery without + /// acknowledging it as processed. + Poison, +} + +/// Why a message stopped being redelivered. +#[derive(Debug)] +pub enum PoisonReason { + /// The handler returned [`HandlerVerdict::Poison`]. + Verdict, + /// The message exhausted [`RedeliveryPolicy::max_deliveries`] attempts. + RedeliveryExhausted, + /// The handler returned an error on its final attempt. + HandlerError(HandlerError), + /// The handler panicked on its final attempt. + Panic, +} + +/// Handles one durable-consumer message at a time. +pub trait MessageHandler: Send { + /// Error returned by [`MessageHandler::handle`]. + type Error: std::error::Error + Send + Sync + 'static; + + /// Handles one message, returning the verdict that drives its + /// acknowledgement. + fn handle( + &mut self, + message: &jetstream::Message, + ) -> impl std::future::Future> + Send; + + /// Called when a message is terminated (stops being redelivered) + /// instead of acknowledged. + /// + /// This is the observability hook for terminal outcomes; the default + /// implementation does nothing. + fn on_poison(&mut self, reason: PoisonReason) -> impl std::future::Future + Send { + let _ = reason; + std::future::ready(()) + } +} + +/// Bounds how many times a message is redelivered before it is poisoned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RedeliveryPolicy { + max_deliveries: u32, + base_delay: Duration, + max_delay: Duration, +} + +impl RedeliveryPolicy { + /// Creates a redelivery policy with exponential backoff bounded by `max_delay`. + pub const fn new(max_deliveries: u32, base_delay: Duration, max_delay: Duration) -> Self { + Self { + max_deliveries, + base_delay, + max_delay, + } + } + + /// Returns the maximum number of deliveries before a message is poisoned. + pub const fn max_deliveries(&self) -> u32 { + self.max_deliveries + } + + /// `already_delivered` counts deliveries prior to the one currently + /// being settled (0 on the first delivery's failure), not the raw + /// JetStream delivery count, which is 1-indexed. + fn decide(self, already_delivered: u32) -> RedeliveryDecision { + if already_delivered.saturating_add(1) >= self.max_deliveries { + return RedeliveryDecision::Poison; + } + let exponent = already_delivered.min(31); + let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); + let delay = self.base_delay.saturating_mul(multiplier).min(self.max_delay); + RedeliveryDecision::Retry { delay } + } +} + +impl Default for RedeliveryPolicy { + fn default() -> Self { + Self::new(5, Duration::from_millis(200), Duration::from_secs(30)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RedeliveryDecision { + Retry { delay: Duration }, + Poison, +} + +enum Outcome { + Ack, + Retry { delay: Duration }, + Term(PoisonReason), +} + +/// Error raised by [`Processor::run`]. +#[derive(Debug, thiserror::Error)] +pub enum ProcessorError { + /// The durable consumer's ack policy is not [`AckPolicy::Explicit`], so + /// ack/nak/term settlement would be meaningless. + #[error("durable consumer '{durable_name}' must use AckPolicy::Explicit")] + InvalidAckPolicy { + /// Name of the durable consumer. + durable_name: String, + }, + /// Provisioning the durable consumer failed for a reason other than an + /// existing consumer with a different configuration. + #[error("failed to provision durable consumer '{durable_name}'")] + ProvisionConsumer { + /// Name of the durable consumer. + durable_name: String, + /// Underlying `async_nats` error. + #[source] + source: ConsumerCreateStrictError, + }, + /// The durable consumer already existed with a configuration that + /// diverges from the one requested. + #[error("durable consumer '{durable_name}' already exists with a different configuration")] + ConsumerConfigMismatch { + /// Name of the durable consumer. + durable_name: String, + }, + /// Opening the message stream for the durable consumer failed. + #[error("failed to open message stream for durable consumer '{durable_name}'")] + OpenMessages { + /// Name of the durable consumer. + durable_name: String, + /// Underlying `async_nats` error. + #[source] + source: async_nats::Error, + }, + /// Reading the next message from the durable consumer failed. + #[error("failed to read next message from durable consumer '{durable_name}'")] + ReadMessage { + /// Name of the durable consumer. + durable_name: String, + /// Underlying `async_nats` error. + #[source] + source: async_nats::Error, + }, + /// Reading a message's delivery metadata failed. + #[error("failed to read delivery metadata for a message")] + ReadMessageInfo { + /// Underlying `async_nats` error. + #[source] + source: async_nats::Error, + }, + /// Acknowledging, nak'ing, or terminating a message failed. + #[error("failed to settle a message")] + Settle { + /// Underlying `async_nats` error. + #[source] + source: async_nats::Error, + }, +} + +async fn provision_consumer( + stream: &jetstream::stream::Stream, + config: pull::Config, +) -> Result { + let durable_name = config.durable_name.clone().unwrap_or_default(); + if config.ack_policy != AckPolicy::Explicit { + return Err(ProcessorError::InvalidAckPolicy { durable_name }); + } + match stream.create_consumer_strict(config).await { + Ok(consumer) => Ok(consumer), + Err(source) if matches!(source.kind(), ConsumerCreateStrictErrorKind::AlreadyExists) => { + Err(ProcessorError::ConsumerConfigMismatch { durable_name }) + } + Err(source) => Err(ProcessorError::ProvisionConsumer { durable_name, source }), + } +} + +/// Drives a durable pull consumer's message loop for one [`MessageHandler`]. +pub struct Processor { + stream: jetstream::stream::Stream, + consumer_config: pull::Config, + redelivery_policy: RedeliveryPolicy, + handler: Handler, +} + +impl Processor +where + Handler: MessageHandler, +{ + /// Creates a processor over `stream` with a durable consumer described + /// by `consumer_config`. + /// + /// `consumer_config.ack_policy` must be [`AckPolicy::Explicit`]; any + /// other policy is rejected by [`Processor::run`] before the consumer is + /// provisioned. + pub fn new(stream: jetstream::stream::Stream, consumer_config: pull::Config, handler: Handler) -> Self { + Self { + stream, + consumer_config, + redelivery_policy: RedeliveryPolicy::default(), + handler, + } + } + + /// Overrides the default [`RedeliveryPolicy`]. + #[must_use] + pub fn with_redelivery_policy(mut self, redelivery_policy: RedeliveryPolicy) -> Self { + self.redelivery_policy = redelivery_policy; + self + } + + /// Provisions the durable consumer and runs the pull loop until + /// `shutdown` is cancelled, the message stream ends, or a fatal error + /// occurs. + pub async fn run(mut self, shutdown: CancellationToken) -> Result<(), ProcessorError> { + let durable_name = self.consumer_config.durable_name.clone().unwrap_or_default(); + let consumer = provision_consumer(&self.stream, self.consumer_config.clone()).await?; + let mut messages = consumer + .messages() + .await + .map_err(|source| ProcessorError::OpenMessages { + durable_name: durable_name.clone(), + source: source.into(), + })?; + + loop { + tokio::select! { + () = shutdown.cancelled() => return Ok(()), + next = messages.next() => { + let Some(message) = next else { + return Ok(()); + }; + let message = message.map_err(|source| ProcessorError::ReadMessage { + durable_name: durable_name.clone(), + source: source.into(), + })?; + self.settle(&message).await?; + } + } + } + } + + async fn settle(&mut self, message: &jetstream::Message) -> Result<(), ProcessorError> { + let info = message + .info() + .map_err(|source| ProcessorError::ReadMessageInfo { source })?; + let already_delivered = u32::try_from(info.delivered.max(1)) + .unwrap_or(u32::MAX) + .saturating_sub(1); + + let outcome = match AssertUnwindSafe(self.handler.handle(message)).catch_unwind().await { + Ok(Ok(HandlerVerdict::Ack)) => Outcome::Ack, + Ok(Ok(HandlerVerdict::Poison)) => Outcome::Term(PoisonReason::Verdict), + Ok(Ok(HandlerVerdict::Retry)) => match self.redelivery_policy.decide(already_delivered) { + RedeliveryDecision::Retry { delay } => Outcome::Retry { delay }, + RedeliveryDecision::Poison => Outcome::Term(PoisonReason::RedeliveryExhausted), + }, + Ok(Err(error)) => match self.redelivery_policy.decide(already_delivered) { + RedeliveryDecision::Retry { delay } => Outcome::Retry { delay }, + RedeliveryDecision::Poison => Outcome::Term(PoisonReason::HandlerError(error)), + }, + Err(_panic) => match self.redelivery_policy.decide(already_delivered) { + RedeliveryDecision::Retry { delay } => Outcome::Retry { delay }, + RedeliveryDecision::Poison => Outcome::Term(PoisonReason::Panic), + }, + }; + + match outcome { + Outcome::Ack => message + .ack() + .await + .map_err(|source| ProcessorError::Settle { source })?, + Outcome::Retry { delay } => message + .ack_with(AckKind::Nak(Some(delay))) + .await + .map_err(|source| ProcessorError::Settle { source })?, + Outcome::Term(reason) => { + message + .ack_with(AckKind::Term) + .await + .map_err(|source| ProcessorError::Settle { source })?; + self.handler.on_poison(reason).await; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-nats/src/processor/tests.rs b/rsworkspace/crates/trogon-decider-nats/src/processor/tests.rs new file mode 100644 index 000000000..fa327d487 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/processor/tests.rs @@ -0,0 +1,420 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_nats::jetstream; +use async_nats::jetstream::consumer::{AckPolicy, pull}; +use bytes::Bytes; +use testcontainers_modules::nats::{Nats, NatsServerCmd}; +use testcontainers_modules::testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner}; +use tokio_util::sync::CancellationToken; + +use super::{HandlerVerdict, MessageHandler, PoisonReason, Processor, RedeliveryDecision, RedeliveryPolicy}; + +#[test] +fn redelivery_policy_retries_with_bounded_exponential_backoff() { + let policy = RedeliveryPolicy::new(3, Duration::from_millis(100), Duration::from_secs(1)); + + assert_eq!( + policy.decide(0), + RedeliveryDecision::Retry { + delay: Duration::from_millis(100) + } + ); + assert_eq!( + policy.decide(1), + RedeliveryDecision::Retry { + delay: Duration::from_millis(200) + } + ); + assert_eq!(policy.decide(2), RedeliveryDecision::Poison); +} + +#[test] +fn redelivery_policy_caps_delay_at_max_delay() { + let policy = RedeliveryPolicy::new(10, Duration::from_millis(100), Duration::from_millis(250)); + + assert_eq!( + policy.decide(5), + RedeliveryDecision::Retry { + delay: Duration::from_millis(250) + } + ); +} + +#[test] +fn redelivery_policy_reports_max_deliveries() { + let policy = RedeliveryPolicy::new(7, Duration::from_millis(1), Duration::from_millis(1)); + + assert_eq!(policy.max_deliveries(), 7); +} + +struct NatsServer { + _container: ContainerAsync, + url: String, +} + +impl NatsServer { + async fn start() -> Self { + let cmd = NatsServerCmd::default().with_jetstream(); + let container = Nats::default() + .with_cmd(&cmd) + .start() + .await + .expect("start NATS testcontainer for processor tests"); + let host = container.get_host().await.expect("get NATS testcontainer host"); + let port = container + .get_host_port_ipv4(4222) + .await + .expect("get NATS testcontainer port"); + Self { + _container: container, + url: format!("{host}:{port}"), + } + } + + fn url(&self) -> &str { + &self.url + } +} + +async fn connect(server: &NatsServer) -> jetstream::Context { + let client = async_nats::ConnectOptions::new() + .connection_timeout(Duration::from_secs(2)) + .connect(server.url()) + .await + .expect("connect to NATS testcontainer"); + jetstream::new(client) +} + +async fn create_events_stream( + js: &jetstream::Context, + name: &str, + subject_wildcard: &str, +) -> jetstream::stream::Stream { + js.create_stream(jetstream::stream::Config { + name: name.to_string(), + subjects: vec![subject_wildcard.to_string()], + allow_atomic_publish: true, + ..Default::default() + }) + .await + .expect("create test events stream") +} + +async fn publish(js: &jetstream::Context, subject: &str, payload: &str) { + js.publish(subject.to_string(), Bytes::from(payload.to_string())) + .await + .expect("send publish") + .await + .expect("ack publish"); +} + +fn consumer_config(durable_name: &str) -> pull::Config { + pull::Config { + durable_name: Some(durable_name.to_string()), + ack_policy: AckPolicy::Explicit, + ack_wait: Duration::from_secs(10), + ..Default::default() + } +} + +async fn wait_until(timeout: Duration, mut condition: impl FnMut() -> bool) { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if condition() { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "condition not met within timeout" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + +#[derive(Debug, thiserror::Error)] +#[error("scripted handler failure")] +struct TestHandlerError; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PoisonRecord { + Verdict, + RedeliveryExhausted, + HandlerError, + Panic, +} + +#[derive(Clone)] +struct RecordingHandler { + verdicts: Arc>, + calls: Arc, + poisoned: Arc>>, +} + +impl RecordingHandler { + fn new(verdicts: Vec) -> Self { + assert!(!verdicts.is_empty(), "scripted handler needs at least one verdict"); + Self { + verdicts: Arc::new(verdicts), + calls: Arc::new(AtomicUsize::new(0)), + poisoned: Arc::new(Mutex::new(Vec::new())), + } + } + + fn call_count(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } + + fn poisoned_records(&self) -> Vec { + self.poisoned.lock().expect("poisoned records lock").clone() + } +} + +impl MessageHandler for RecordingHandler { + type Error = TestHandlerError; + + async fn handle(&mut self, _message: &jetstream::Message) -> Result { + let index = self.calls.fetch_add(1, Ordering::SeqCst); + let verdict = self + .verdicts + .get(index) + .copied() + .unwrap_or_else(|| *self.verdicts.last().expect("at least one scripted verdict")); + Ok(verdict) + } + + async fn on_poison(&mut self, reason: PoisonReason) { + let record = match reason { + PoisonReason::Verdict => PoisonRecord::Verdict, + PoisonReason::RedeliveryExhausted => PoisonRecord::RedeliveryExhausted, + PoisonReason::HandlerError(_) => PoisonRecord::HandlerError, + PoisonReason::Panic => PoisonRecord::Panic, + }; + self.poisoned.lock().expect("poisoned records lock").push(record); + } +} + +#[tokio::test] +async fn processor_acks_message_and_never_redelivers_it() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_ACK", "processor.ack.>").await; + publish(&js, "processor.ack.one", "payload").await; + + let handler = RecordingHandler::new(vec![HandlerVerdict::Ack]); + let processor = Processor::new(stream, consumer_config("processor-ack"), handler.clone()).with_redelivery_policy( + RedeliveryPolicy::new(5, Duration::from_millis(20), Duration::from_millis(100)), + ); + let shutdown = CancellationToken::new(); + let run_shutdown = shutdown.clone(); + let run_handle = tokio::spawn(async move { processor.run(run_shutdown).await }); + + wait_until(Duration::from_secs(5), || handler.call_count() >= 1).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(handler.call_count(), 1, "an acked message must not be redelivered"); + assert!(handler.poisoned_records().is_empty()); + + shutdown.cancel(); + run_handle + .await + .expect("processor task should not panic") + .expect("processor should shut down cleanly"); +} + +#[tokio::test] +async fn processor_naks_with_backoff_and_redelivers_until_ack() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_RETRY", "processor.retry.>").await; + publish(&js, "processor.retry.one", "payload").await; + + let handler = RecordingHandler::new(vec![HandlerVerdict::Retry, HandlerVerdict::Retry, HandlerVerdict::Ack]); + let processor = Processor::new(stream, consumer_config("processor-retry"), handler.clone()).with_redelivery_policy( + RedeliveryPolicy::new(5, Duration::from_millis(30), Duration::from_millis(100)), + ); + let shutdown = CancellationToken::new(); + let run_shutdown = shutdown.clone(); + let run_handle = tokio::spawn(async move { processor.run(run_shutdown).await }); + + wait_until(Duration::from_secs(5), || handler.call_count() >= 3).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + handler.call_count(), + 3, + "the message should stop being redelivered once acked" + ); + assert!(handler.poisoned_records().is_empty()); + + shutdown.cancel(); + run_handle + .await + .expect("processor task should not panic") + .expect("processor should shut down cleanly"); +} + +#[tokio::test] +async fn processor_poisons_message_on_explicit_poison_verdict() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_POISON", "processor.poison.>").await; + publish(&js, "processor.poison.one", "payload").await; + + let handler = RecordingHandler::new(vec![HandlerVerdict::Poison]); + let processor = + Processor::new(stream, consumer_config("processor-poison"), handler.clone()).with_redelivery_policy( + RedeliveryPolicy::new(5, Duration::from_millis(30), Duration::from_millis(100)), + ); + let shutdown = CancellationToken::new(); + let run_shutdown = shutdown.clone(); + let run_handle = tokio::spawn(async move { processor.run(run_shutdown).await }); + + wait_until(Duration::from_secs(5), || !handler.poisoned_records().is_empty()).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + handler.call_count(), + 1, + "an explicitly poisoned message must not be redelivered" + ); + assert_eq!(handler.poisoned_records(), vec![PoisonRecord::Verdict]); + + shutdown.cancel(); + run_handle + .await + .expect("processor task should not panic") + .expect("processor should shut down cleanly"); +} + +#[tokio::test] +async fn processor_poisons_message_after_exhausting_redeliveries() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_EXHAUST", "processor.exhaust.>").await; + publish(&js, "processor.exhaust.one", "payload").await; + + let handler = RecordingHandler::new(vec![HandlerVerdict::Retry]); + let processor = + Processor::new(stream, consumer_config("processor-exhaust"), handler.clone()).with_redelivery_policy( + RedeliveryPolicy::new(3, Duration::from_millis(20), Duration::from_millis(50)), + ); + let shutdown = CancellationToken::new(); + let run_shutdown = shutdown.clone(); + let run_handle = tokio::spawn(async move { processor.run(run_shutdown).await }); + + wait_until(Duration::from_secs(5), || !handler.poisoned_records().is_empty()).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(handler.poisoned_records(), vec![PoisonRecord::RedeliveryExhausted]); + assert!( + handler.call_count() >= 1 && handler.call_count() <= 3, + "delivery count should be bounded by max_deliveries, got {}", + handler.call_count() + ); + + shutdown.cancel(); + run_handle + .await + .expect("processor task should not panic") + .expect("processor should shut down cleanly"); +} + +#[tokio::test] +async fn processor_poisons_message_on_handler_error_after_exhausting_redeliveries() { + struct FailingHandler { + calls: Arc, + poisoned: Arc>>, + } + + impl MessageHandler for FailingHandler { + type Error = TestHandlerError; + + async fn handle(&mut self, _message: &jetstream::Message) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + Err(TestHandlerError) + } + + async fn on_poison(&mut self, reason: PoisonReason) { + let record = match reason { + PoisonReason::HandlerError(_) => PoisonRecord::HandlerError, + PoisonReason::Verdict => PoisonRecord::Verdict, + PoisonReason::RedeliveryExhausted => PoisonRecord::RedeliveryExhausted, + PoisonReason::Panic => PoisonRecord::Panic, + }; + self.poisoned.lock().expect("poisoned records lock").push(record); + } + } + + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_HANDLER_ERROR", "processor.handler_error.>").await; + publish(&js, "processor.handler_error.one", "payload").await; + + let calls = Arc::new(AtomicUsize::new(0)); + let poisoned = Arc::new(Mutex::new(Vec::new())); + let handler = FailingHandler { + calls: calls.clone(), + poisoned: poisoned.clone(), + }; + let processor = Processor::new(stream, consumer_config("processor-handler-error"), handler).with_redelivery_policy( + RedeliveryPolicy::new(2, Duration::from_millis(20), Duration::from_millis(50)), + ); + let shutdown = CancellationToken::new(); + let run_shutdown = shutdown.clone(); + let run_handle = tokio::spawn(async move { processor.run(run_shutdown).await }); + + wait_until(Duration::from_secs(5), || { + !poisoned.lock().expect("poisoned records lock").is_empty() + }) + .await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + poisoned.lock().expect("poisoned records lock").clone(), + vec![PoisonRecord::HandlerError] + ); + + shutdown.cancel(); + run_handle + .await + .expect("processor task should not panic") + .expect("processor should shut down cleanly"); +} + +#[tokio::test] +async fn processor_stops_cleanly_on_shutdown() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_SHUTDOWN", "processor.shutdown.>").await; + + let handler = RecordingHandler::new(vec![HandlerVerdict::Ack]); + let processor = Processor::new(stream, consumer_config("processor-shutdown"), handler); + let shutdown = CancellationToken::new(); + let run_shutdown = shutdown.clone(); + let run_handle = tokio::spawn(async move { processor.run(run_shutdown).await }); + + tokio::time::sleep(Duration::from_millis(100)).await; + shutdown.cancel(); + + let result = tokio::time::timeout(Duration::from_secs(5), run_handle) + .await + .expect("processor should shut down promptly after cancellation") + .expect("processor task should not panic"); + assert!(result.is_ok()); +} + +#[tokio::test] +async fn processor_rejects_non_explicit_ack_policy() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROCESSOR_ACK_POLICY", "processor.ack_policy.>").await; + + let mut config = consumer_config("processor-ack-policy"); + config.ack_policy = AckPolicy::None; + let handler = RecordingHandler::new(vec![HandlerVerdict::Ack]); + let processor = Processor::new(stream, config, handler); + + let error = processor + .run(CancellationToken::new()) + .await + .expect_err("non-explicit ack policy must be rejected"); + + assert!(matches!(error, super::ProcessorError::InvalidAckPolicy { .. })); +} diff --git a/rsworkspace/crates/trogon-decider-nats/src/projector.rs b/rsworkspace/crates/trogon-decider-nats/src/projector.rs new file mode 100644 index 000000000..e2f98efed --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/projector.rs @@ -0,0 +1,262 @@ +//! Generic catch-up driver for subject-filtered JetStream projections. +//! +//! [`Projector`] owns the parts of a read-model catch-up that stay identical +//! across backends: creating the subject-filtered ordered consumer (via +//! [`crate::stream_store`]'s replay helpers), finding the stream's current +//! tail, and applying events in order. What differs across backends is where +//! the checkpoint lives and how it advances, which [`ProjectionCheckpointStore`] +//! captures behind one contract: +//! +//! - A NATS KV backed checkpoint has no transaction to piggyback on, so the +//! driver calls [`ProjectionCheckpointStore::save`] after every applied +//! event and that call does the real CAS write. +//! - A Postgres-backed (or any externally transactional) checkpoint advances +//! atomically with the projection write inside +//! [`ProjectionApply::apply`]: the closure commits both in one transaction +//! and returns the checkpoint it just committed, and the corresponding +//! `save` implementation is a no-op because the write already happened. +//! +//! Both shapes fit the same loop because the apply step, not the driver, +//! decides what checkpoint value to record next. +//! +//! [`Projector::catch_up`] resumes from whatever [`ProjectionCheckpointStore::load`] +//! returns. A caller that wants to always reconcile from scratch (ignoring any +//! stored checkpoint) can compose that by implementing `load` to always return +//! [`CheckpointSequence::NONE`]; the driver itself has no such policy baked in. +//! +//! The stream tail used as the catch-up target is captured once, at the start +//! of the call, via a fresh [`jetstream::stream::Stream::get_info`]. Events +//! published after that point are not included; call [`Projector::catch_up`] +//! again to pick them up. + +use async_nats::jetstream; +use trogon_decider_runtime::StreamEvent; + +use crate::stream_store::{ReadStreamError, StreamStoreError, read_stream_range, read_subject_stream}; + +/// A projection checkpoint position. +/// +/// `CheckpointSequence` wraps the JetStream stream sequence up to and +/// including which a projection has been applied. [`CheckpointSequence::NONE`] +/// means no event has been applied yet, distinct from sequence `1` (the first +/// possible event). +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CheckpointSequence(u64); + +impl CheckpointSequence { + /// No projection progress has been recorded yet. + pub const NONE: Self = Self(0); + + /// Wraps an already known checkpoint sequence. + pub const fn new(sequence: u64) -> Self { + Self(sequence) + } + + /// Returns the checkpoint as a plain stream sequence. + pub const fn as_u64(self) -> u64 { + self.0 + } + + /// Returns whether no projection progress has been recorded yet. + pub const fn is_none(self) -> bool { + self.0 == 0 + } + + /// Returns the stream sequence a catch-up should resume from. + /// + /// This is the checkpoint's sequence plus one: [`CheckpointSequence::NONE`] + /// resumes from sequence `1`, the first possible event. + pub const fn next_from_sequence(self) -> u64 { + self.0.saturating_add(1) + } +} + +impl From for CheckpointSequence { + fn from(position: trogon_decider_runtime::StreamPosition) -> Self { + Self(position.as_u64()) + } +} + +/// Loads and saves a projection's checkpoint. +/// +/// See the module documentation for how this contract fits both a +/// NATS KV owned checkpoint and an externally transactional one. +pub trait ProjectionCheckpointStore: Send + Sync { + /// Error raised while loading or saving the checkpoint. + type Error: std::error::Error + Send + Sync + 'static; + + /// Loads the checkpoint to resume a catch-up from. + fn load(&self) -> impl std::future::Future> + Send; + + /// Records a new checkpoint after an event has been applied. + /// + /// Implementations backed by an externally transactional checkpoint (for + /// example, one advanced inside the same Postgres transaction as the + /// projection write) should make this a no-op: the checkpoint was already + /// committed by [`ProjectionApply::apply`]. + fn save(&self, checkpoint: CheckpointSequence) + -> impl std::future::Future> + Send; +} + +/// Applies one projected event and returns the checkpoint to record for it. +/// +/// The returned [`CheckpointSequence`] is what [`Projector`] passes to +/// [`ProjectionCheckpointStore::save`] immediately afterward. Implementations +/// that commit their own checkpoint transactionally alongside the projection +/// write should still return the sequence they committed, even though the +/// paired `save` call will be a no-op. +pub trait ProjectionApply: Send { + /// Error raised while applying a projected event. + type Error: std::error::Error + Send + Sync + 'static; + + /// Applies one event, returning the checkpoint to record for it. + fn apply( + &mut self, + event: StreamEvent, + ) -> impl std::future::Future> + Send; +} + +/// Error raised by [`Projector::catch_up`]. +#[derive(Debug, thiserror::Error)] +pub enum CatchUpError { + /// Loading the checkpoint to resume from failed. + #[error("failed to load projection checkpoint: {0}")] + LoadCheckpoint(#[source] CheckpointError), + /// Querying the stream's current tail failed. + #[error("failed to query stream tail: {0}")] + QueryTail(#[source] StreamStoreError), + /// Replaying stream events failed. + #[error("failed to replay stream events: {0}")] + Replay(#[source] StreamStoreError), + /// Applying a projected event failed. + #[error("failed to apply projected event at sequence {sequence}: {source}")] + Apply { + /// Stream sequence of the event that failed to apply. + sequence: u64, + /// Underlying application error. + #[source] + source: ApplyError, + }, + /// Saving the checkpoint after applying an event failed. + #[error("failed to save projection checkpoint: {0}")] + SaveCheckpoint(#[source] CheckpointError), +} + +/// Result of one [`Projector::catch_up`] call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CatchUpOutcome { + /// Number of events applied during this call. + pub events_applied: usize, + /// Checkpoint recorded after the call completed. + pub checkpoint: CheckpointSequence, + /// Whether the replay reached the stream tail observed at the start of the call. + /// + /// This is `true` on every successful return: a stream ending before the + /// observed tail is a transient condition retried internally, and only + /// gives up (returning [`CatchUpError::Replay`]) after exhausting + /// retries. The field stays meaningful as a caller-facing signal for + /// "did this call make full progress", separate from how that guarantee + /// is implemented. + pub reached_target: bool, +} + +/// Drives catch-up for one subject-filtered projection over a JetStream stream. +pub struct Projector { + stream: jetstream::stream::Stream, + projection_id: String, + filter_subject: Option, + checkpoint: Checkpoint, +} + +impl Projector +where + Checkpoint: ProjectionCheckpointStore, +{ + /// Creates a projector over the full stream, applying every event regardless of subject. + pub fn new(stream: jetstream::stream::Stream, projection_id: impl Into, checkpoint: Checkpoint) -> Self { + Self { + stream, + projection_id: projection_id.into(), + filter_subject: None, + checkpoint, + } + } + + /// Restricts catch-up to events published on `filter_subject`. + #[must_use] + pub fn with_filter_subject(mut self, filter_subject: impl Into) -> Self { + self.filter_subject = Some(filter_subject.into()); + self + } + + /// Returns the checkpoint store backing this projector. + pub fn checkpoint_store(&self) -> &Checkpoint { + &self.checkpoint + } + + /// Catches a projection up to the stream tail observed when this call started. + /// + /// Resumes from whatever [`ProjectionCheckpointStore::load`] returns. + /// Applies events in stream order, saving the checkpoint after each one. + pub async fn catch_up( + &self, + mut apply: Apply, + ) -> Result> + where + Apply: ProjectionApply, + { + let checkpoint = self.checkpoint.load().await.map_err(CatchUpError::LoadCheckpoint)?; + let from_sequence = checkpoint.next_from_sequence(); + + let info = self + .stream + .get_info() + .await + .map_err(|source| CatchUpError::QueryTail(ReadStreamError::QueryStreamInfo { source }.into()))?; + let to_sequence = info.state.last_sequence; + + let events = self + .replay(from_sequence, to_sequence) + .await + .map_err(CatchUpError::Replay)?; + + let events_applied = events.len(); + let mut checkpoint = checkpoint; + for event in events { + let sequence = event.stream_position.as_u64(); + checkpoint = apply + .apply(event) + .await + .map_err(|source| CatchUpError::Apply { sequence, source })?; + self.checkpoint + .save(checkpoint) + .await + .map_err(CatchUpError::SaveCheckpoint)?; + } + + Ok(CatchUpOutcome { + events_applied, + checkpoint, + reached_target: true, + }) + } + + async fn replay(&self, from_sequence: u64, to_sequence: u64) -> Result, StreamStoreError> { + match self.filter_subject.as_deref() { + Some(subject) => { + read_subject_stream( + &self.stream, + self.projection_id.as_str(), + subject, + from_sequence, + to_sequence, + ) + .await + } + None => read_stream_range(&self.stream, from_sequence, to_sequence).await, + } + } +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-nats/src/projector/tests.rs b/rsworkspace/crates/trogon-decider-nats/src/projector/tests.rs new file mode 100644 index 000000000..f2ec87fdb --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/projector/tests.rs @@ -0,0 +1,254 @@ +use std::sync::Arc; +use std::time::Duration; + +use async_nats::jetstream; +use testcontainers_modules::nats::{Nats, NatsServerCmd}; +use testcontainers_modules::testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner}; +use tokio::sync::Mutex; +use trogon_decider_runtime::{Event, EventId, Headers, StreamEvent}; +use uuid::Uuid; + +use crate::append_stream; +use crate::stream_store::StreamSubject; + +use super::{CheckpointSequence, ProjectionApply, ProjectionCheckpointStore, Projector}; + +struct NatsServer { + _container: ContainerAsync, + url: String, +} + +impl NatsServer { + async fn start() -> Self { + let cmd = NatsServerCmd::default().with_jetstream(); + let container = Nats::default() + .with_cmd(&cmd) + .start() + .await + .expect("start NATS testcontainer for projector tests"); + let host = container.get_host().await.expect("get NATS testcontainer host"); + let port = container + .get_host_port_ipv4(4222) + .await + .expect("get NATS testcontainer port"); + Self { + _container: container, + url: format!("{host}:{port}"), + } + } + + fn url(&self) -> &str { + &self.url + } +} + +async fn connect(server: &NatsServer) -> jetstream::Context { + let client = async_nats::ConnectOptions::new() + .connection_timeout(Duration::from_secs(2)) + .connect(server.url()) + .await + .expect("connect to NATS testcontainer"); + jetstream::new(client) +} + +async fn create_events_stream( + js: &jetstream::Context, + name: &str, + subject_wildcard: &str, +) -> jetstream::stream::Stream { + js.create_stream(jetstream::stream::Config { + name: name.to_string(), + subjects: vec![subject_wildcard.to_string()], + allow_atomic_publish: true, + ..Default::default() + }) + .await + .expect("create test events stream") +} + +fn make_event(id: u128, content: &[u8]) -> Event { + Event { + id: EventId::from(Uuid::from_u128(id)), + r#type: "test.event.v1".to_string(), + content: content.to_vec(), + headers: Headers::empty(), + } +} + +fn make_subject(value: &str) -> StreamSubject { + StreamSubject::new(value).expect("subject must be valid") +} + +struct FakeCheckpointStore { + checkpoint: Mutex, +} + +impl FakeCheckpointStore { + fn new(initial: CheckpointSequence) -> Self { + Self { + checkpoint: Mutex::new(initial), + } + } + + async fn current(&self) -> CheckpointSequence { + *self.checkpoint.lock().await + } +} + +impl ProjectionCheckpointStore for FakeCheckpointStore { + type Error = std::convert::Infallible; + + async fn load(&self) -> Result { + Ok(*self.checkpoint.lock().await) + } + + async fn save(&self, checkpoint: CheckpointSequence) -> Result<(), Self::Error> { + *self.checkpoint.lock().await = checkpoint; + Ok(()) + } +} + +#[derive(Clone, Default)] +struct RecordingApply { + applied: Arc>>, +} + +impl RecordingApply { + async fn applied(&self) -> Vec { + self.applied.lock().await.clone() + } +} + +impl ProjectionApply for RecordingApply { + type Error = std::convert::Infallible; + + async fn apply(&mut self, event: StreamEvent) -> Result { + let checkpoint = CheckpointSequence::from(event.stream_position); + self.applied.lock().await.push(event); + Ok(checkpoint) + } +} + +#[test] +fn checkpoint_sequence_none_resumes_from_first_sequence() { + assert_eq!(CheckpointSequence::NONE.next_from_sequence(), 1); + assert!(CheckpointSequence::NONE.is_none()); +} + +#[test] +fn checkpoint_sequence_resumes_right_after_itself() { + let checkpoint = CheckpointSequence::new(7); + assert_eq!(checkpoint.next_from_sequence(), 8); + assert!(!checkpoint.is_none()); +} + +#[tokio::test] +async fn catch_up_applies_all_events_from_zero() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROJECTOR_FROM_ZERO", "projector.from_zero.>").await; + let subject = make_subject("projector.from_zero.alpha"); + for index in 0..4u128 { + append_stream(&js, subject.clone(), None, &[make_event(index, b"payload")]) + .await + .expect("publish event"); + } + + let checkpoint_store = FakeCheckpointStore::new(CheckpointSequence::NONE); + let projector = Projector::new(stream, "projection", checkpoint_store); + + let outcome = projector + .catch_up(RecordingApply::default()) + .await + .expect("catch up should succeed"); + + assert_eq!(outcome.events_applied, 4); + assert_eq!(outcome.checkpoint, CheckpointSequence::new(4)); + assert!(outcome.reached_target); + assert_eq!(projector.checkpoint_store().current().await, CheckpointSequence::new(4)); +} + +#[tokio::test] +async fn catch_up_resumes_from_existing_checkpoint() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROJECTOR_RESUME", "projector.resume.>").await; + let subject = make_subject("projector.resume.alpha"); + let mut positions = Vec::new(); + for index in 0..5u128 { + let position = append_stream(&js, subject.clone(), None, &[make_event(index, b"payload")]) + .await + .expect("publish event"); + positions.push(position); + } + + let checkpoint_store = FakeCheckpointStore::new(CheckpointSequence::from(positions[1])); + let projector = Projector::new(stream, "projection", checkpoint_store); + + let apply = RecordingApply::default(); + let outcome = projector + .catch_up(apply.clone()) + .await + .expect("catch up should succeed"); + let applied = apply.applied().await; + + assert_eq!(outcome.events_applied, 3); + assert_eq!(applied.len(), 3); + assert_eq!(applied[0].stream_position, positions[2]); + assert_eq!(outcome.checkpoint, CheckpointSequence::from(*positions.last().unwrap())); +} + +#[tokio::test] +async fn catch_up_filters_by_subject() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROJECTOR_FILTER", "projector.filter.>").await; + let alpha = make_subject("projector.filter.alpha"); + let beta = make_subject("projector.filter.beta"); + + append_stream(&js, alpha.clone(), None, &[make_event(1, b"alpha-1")]) + .await + .expect("publish alpha event"); + append_stream(&js, beta.clone(), None, &[make_event(2, b"beta-1")]) + .await + .expect("publish beta event"); + append_stream(&js, alpha.clone(), None, &[make_event(3, b"alpha-2")]) + .await + .expect("publish alpha event"); + + let checkpoint_store = FakeCheckpointStore::new(CheckpointSequence::NONE); + let projector = Projector::new(stream, "alpha-projection", checkpoint_store).with_filter_subject(alpha.to_string()); + + let apply = RecordingApply::default(); + let outcome = projector + .catch_up(apply.clone()) + .await + .expect("catch up should succeed"); + let applied = apply.applied().await; + + assert_eq!(outcome.events_applied, 2); + assert!(applied.iter().all(|event| event.stream_id == "alpha-projection")); +} + +#[tokio::test] +async fn catch_up_reports_no_new_events_when_checkpoint_is_at_tail() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "PROJECTOR_AT_TAIL", "projector.at_tail.>").await; + let subject = make_subject("projector.at_tail.alpha"); + let position = append_stream(&js, subject, None, &[make_event(1, b"payload")]) + .await + .expect("publish event"); + + let checkpoint_store = FakeCheckpointStore::new(CheckpointSequence::from(position)); + let projector = Projector::new(stream, "projection", checkpoint_store); + + let outcome = projector + .catch_up(RecordingApply::default()) + .await + .expect("catch up should succeed"); + + assert_eq!(outcome.events_applied, 0); + assert_eq!(outcome.checkpoint, CheckpointSequence::from(position)); + assert!(outcome.reached_target); +} diff --git a/rsworkspace/crates/trogon-decider-nats/src/provision.rs b/rsworkspace/crates/trogon-decider-nats/src/provision.rs new file mode 100644 index 000000000..bb3f1c0c4 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/provision.rs @@ -0,0 +1,276 @@ +//! Idempotent create-or-open provisioning for JetStream streams and buckets. +//! +//! [`ensure_stream`] and [`ensure_bucket`] give every read-side primitive in +//! this crate the same "create it if it is missing, otherwise validate it +//! matches what the caller needs" entry point. Both functions check for the +//! resource first with a plain get, because JetStream's `STREAM.CREATE` API +//! does not reliably reject a create issued against an already-existing +//! stream: some divergent fields (retention) trigger a `STREAM_NAME_EXIST` +//! conflict, but others (subjects, `allow_atomic_publish`) are silently +//! accepted and applied as an in-place update of the existing stream. +//! Calling create first would therefore risk mutating a stream out from +//! under a caller who only meant to validate it. The get-first check closes +//! that hole; the residual race (two callers creating the same resource for +//! the first time concurrently) is still handled by falling back to a get on +//! an already-exists conflict from create. When the resource already +//! exists, its configuration is validated against the caller's requirements +//! and a [`StreamConfigMismatch`] or [`KvConfigMismatch`] names the specific +//! field that diverged, rather than dumping the whole configuration. + +use async_nats::jetstream; +use async_nats::jetstream::ErrorCode; +use async_nats::jetstream::context::{ + CreateKeyValueError, CreateStreamError, GetStreamError, GetStreamErrorKind, KeyValueError, KeyValueErrorKind, +}; +use async_nats::jetstream::kv; +use async_nats::jetstream::stream::RetentionPolicy; +use trogon_nats::jetstream::{is_create_key_value_already_exists, is_create_stream_already_exists}; + +/// A single divergent field between a required and an existing stream +/// configuration. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum StreamConfigMismatch { + /// The existing stream's retention policy does not match the requirement. + #[error("stream '{stream}' retention mismatch: expected {expected:?}, found {actual:?}")] + Retention { + /// Name of the stream being validated. + stream: String, + /// Retention policy required by the caller. + expected: RetentionPolicy, + /// Retention policy found on the existing stream. + actual: RetentionPolicy, + }, + /// The existing stream's atomic publish setting does not match the requirement. + /// + /// A NATS server older than 2.11 does not persist `allow_atomic_publish` + /// at all, so an existing stream on such a server always reports `false` + /// here regardless of what was requested at creation. + #[error("stream '{stream}' allow_atomic_publish mismatch: expected {expected}, found {actual}")] + AllowAtomicPublish { + /// Name of the stream being validated. + stream: String, + /// Atomic publish setting required by the caller. + expected: bool, + /// Atomic publish setting found on the existing stream. + actual: bool, + }, + /// The existing stream does not carry a subject the caller requires. + #[error("stream '{stream}' is missing required subject '{subject}'")] + MissingSubject { + /// Name of the stream being validated. + stream: String, + /// Subject the caller requires but did not find. + subject: String, + }, +} + +/// A single divergent field between a required and an existing Key/Value +/// bucket configuration. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum KvConfigMismatch { + /// The existing bucket's history depth does not match the requirement. + #[error("bucket '{bucket}' history mismatch: expected {expected}, found {actual}")] + History { + /// Name of the bucket being validated. + bucket: String, + /// History depth required by the caller. + expected: i64, + /// History depth found on the existing bucket. + actual: i64, + }, +} + +/// Error returned by [`ensure_stream`]. +#[derive(Debug, thiserror::Error)] +pub enum EnsureStreamError { + /// Creating the stream failed for a reason other than the stream already existing. + #[error("failed to create stream '{name}'")] + Create { + /// Name of the stream that failed to create. + name: String, + /// Underlying `async_nats` error. + #[source] + source: CreateStreamError, + }, + /// The stream already existed but fetching its current configuration failed. + #[error("failed to get existing stream '{name}'")] + Get { + /// Name of the stream that failed to fetch. + name: String, + /// Underlying `async_nats` error. + #[source] + source: GetStreamError, + }, + /// The stream already existed with a configuration that diverges from the requirement. + #[error(transparent)] + ConfigMismatch(#[from] StreamConfigMismatch), +} + +/// Error returned by [`ensure_bucket`]. +#[derive(Debug, thiserror::Error)] +pub enum EnsureBucketError { + /// Creating the bucket failed for a reason other than the bucket already existing. + #[error("failed to create bucket '{bucket}'")] + Create { + /// Name of the bucket that failed to create. + bucket: String, + /// Underlying `async_nats` error. + #[source] + source: CreateKeyValueError, + }, + /// The bucket already existed but fetching its current configuration failed. + #[error("failed to get existing bucket '{bucket}'")] + Get { + /// Name of the bucket that failed to fetch. + bucket: String, + /// Underlying `async_nats` error. + #[source] + source: KeyValueError, + }, + /// The bucket already existed with a configuration that diverges from the requirement. + #[error(transparent)] + ConfigMismatch(#[from] KvConfigMismatch), +} + +/// Creates a JetStream stream, or opens it if it already exists. +/// +/// `config` is both the configuration used to create the stream and the +/// requirement checked against an already-existing stream. Validation covers +/// [`jetstream::stream::Config::retention`], [`jetstream::stream::Config::allow_atomic_publish`], +/// and subject coverage (every subject in `config.subjects` must already be +/// present on the existing stream); it does not compare every field, so +/// changes to unvalidated fields (limits, storage type, replicas, ...) on an +/// existing stream go unnoticed. Concurrent callers racing to create the same +/// stream both succeed: the loser's create fails with an already-exists +/// error, which this function treats as "someone else already created it" +/// and resolves with a get instead of surfacing an error. +pub async fn ensure_stream( + js: &jetstream::Context, + config: jetstream::stream::Config, +) -> Result { + let name = config.name.clone(); + + match js.get_stream(&name).await { + Ok(stream) => { + validate_stream_config(&name, &config, stream.cached_info())?; + return Ok(stream); + } + Err(source) if is_get_stream_not_found(&source) => {} + Err(source) => return Err(EnsureStreamError::Get { name, source }), + } + + match js.create_stream(config.clone()).await { + Ok(stream) => Ok(stream), + Err(source) if is_create_stream_already_exists(&source) => { + let stream = js.get_stream(&name).await.map_err(|source| EnsureStreamError::Get { + name: name.clone(), + source, + })?; + validate_stream_config(&name, &config, stream.cached_info())?; + Ok(stream) + } + Err(source) => Err(EnsureStreamError::Create { name, source }), + } +} + +/// Creates a Key/Value bucket, or opens it if it already exists. +/// +/// `config` is both the configuration used to create the bucket and the +/// requirement checked against an already-existing bucket. Validation covers +/// [`kv::Config::history`] only; it does not compare every field. Concurrent +/// callers racing to create the same bucket both succeed, following the same +/// get-first-then-create-with-fallback sequencing as [`ensure_stream`]. +pub async fn ensure_bucket(js: &jetstream::Context, config: kv::Config) -> Result { + let bucket = config.bucket.clone(); + + match js.get_key_value(&bucket).await { + Ok(store) => { + validate_kv_config(&bucket, &config, &store.stream.cached_info().config)?; + return Ok(store); + } + Err(source) if is_get_key_value_not_found(&source) => {} + Err(source) => return Err(EnsureBucketError::Get { bucket, source }), + } + + match js.create_key_value(config.clone()).await { + Ok(store) => Ok(store), + Err(source) if is_create_key_value_already_exists(&source) => { + let store = js + .get_key_value(&bucket) + .await + .map_err(|source| EnsureBucketError::Get { + bucket: bucket.clone(), + source, + })?; + validate_kv_config(&bucket, &config, &store.stream.cached_info().config)?; + Ok(store) + } + Err(source) => Err(EnsureBucketError::Create { bucket, source }), + } +} + +fn is_get_stream_not_found(error: &GetStreamError) -> bool { + matches!( + error.kind(), + GetStreamErrorKind::JetStream(ref source) if source.error_code() == ErrorCode::STREAM_NOT_FOUND + ) +} + +fn is_get_key_value_not_found(error: &KeyValueError) -> bool { + if error.kind() != KeyValueErrorKind::GetBucket { + return false; + } + + std::error::Error::source(error) + .and_then(|source| source.downcast_ref::()) + .is_some_and(is_get_stream_not_found) +} + +fn validate_stream_config( + name: &str, + required: &jetstream::stream::Config, + actual: &jetstream::stream::Info, +) -> Result<(), StreamConfigMismatch> { + let actual = &actual.config; + if actual.retention != required.retention { + return Err(StreamConfigMismatch::Retention { + stream: name.to_string(), + expected: required.retention, + actual: actual.retention, + }); + } + if actual.allow_atomic_publish != required.allow_atomic_publish { + return Err(StreamConfigMismatch::AllowAtomicPublish { + stream: name.to_string(), + expected: required.allow_atomic_publish, + actual: actual.allow_atomic_publish, + }); + } + for subject in &required.subjects { + if !actual.subjects.contains(subject) { + return Err(StreamConfigMismatch::MissingSubject { + stream: name.to_string(), + subject: subject.clone(), + }); + } + } + Ok(()) +} + +fn validate_kv_config( + bucket: &str, + required: &kv::Config, + actual: &jetstream::stream::Config, +) -> Result<(), KvConfigMismatch> { + if actual.max_messages_per_subject != required.history { + return Err(KvConfigMismatch::History { + bucket: bucket.to_string(), + expected: required.history, + actual: actual.max_messages_per_subject, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-nats/src/provision/tests.rs b/rsworkspace/crates/trogon-decider-nats/src/provision/tests.rs new file mode 100644 index 000000000..5e3509cd6 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/provision/tests.rs @@ -0,0 +1,238 @@ +use std::time::Duration; + +use async_nats::jetstream; +use async_nats::jetstream::kv; +use async_nats::jetstream::stream::RetentionPolicy; +use testcontainers_modules::nats::{Nats, NatsServerCmd}; +use testcontainers_modules::testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner}; + +use super::{ + EnsureBucketError, EnsureStreamError, KvConfigMismatch, StreamConfigMismatch, ensure_bucket, ensure_stream, +}; + +struct NatsServer { + _container: ContainerAsync, + url: String, +} + +impl NatsServer { + async fn start() -> Self { + let cmd = NatsServerCmd::default().with_jetstream(); + let container = Nats::default() + .with_cmd(&cmd) + .start() + .await + .expect("start NATS testcontainer for provisioning tests"); + let host = container.get_host().await.expect("get NATS testcontainer host"); + let port = container + .get_host_port_ipv4(4222) + .await + .expect("get NATS testcontainer port"); + Self { + _container: container, + url: format!("{host}:{port}"), + } + } + + fn url(&self) -> &str { + &self.url + } +} + +async fn connect(server: &NatsServer) -> jetstream::Context { + let client = async_nats::ConnectOptions::new() + .connection_timeout(Duration::from_secs(2)) + .connect(server.url()) + .await + .expect("connect to NATS testcontainer"); + jetstream::new(client) +} + +fn stream_config(name: &str, subject: &str) -> jetstream::stream::Config { + jetstream::stream::Config { + name: name.to_string(), + subjects: vec![subject.to_string()], + retention: RetentionPolicy::Limits, + allow_atomic_publish: false, + ..Default::default() + } +} + +fn bucket_config(bucket: &str, history: i64) -> kv::Config { + kv::Config { + bucket: bucket.to_string(), + history, + ..Default::default() + } +} + +#[tokio::test] +async fn ensure_stream_creates_a_fresh_stream() { + let server = NatsServer::start().await; + let js = connect(&server).await; + + let stream = ensure_stream(&js, stream_config("FRESH_STREAM", "fresh.stream.>")) + .await + .expect("fresh stream should be created"); + + assert_eq!(stream.cached_info().config.name, "FRESH_STREAM"); +} + +#[tokio::test] +async fn ensure_stream_opens_an_existing_matching_stream() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let config = stream_config("EXISTING_STREAM", "existing.stream.>"); + + ensure_stream(&js, config.clone()) + .await + .expect("first ensure_stream call should create the stream"); + + let stream = ensure_stream(&js, config) + .await + .expect("second ensure_stream call should open the existing stream"); + + assert_eq!(stream.cached_info().config.name, "EXISTING_STREAM"); +} + +#[tokio::test] +async fn ensure_stream_rejects_retention_mismatch() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let mut config = stream_config("RETENTION_STREAM", "retention.stream.>"); + config.retention = RetentionPolicy::Limits; + ensure_stream(&js, config.clone()).await.expect("create stream"); + + config.retention = RetentionPolicy::WorkQueue; + let error = ensure_stream(&js, config) + .await + .expect_err("retention mismatch should be rejected"); + + assert!(matches!( + error, + EnsureStreamError::ConfigMismatch(StreamConfigMismatch::Retention { stream, .. }) + if stream == "RETENTION_STREAM" + )); +} + +#[tokio::test] +async fn ensure_stream_rejects_allow_atomic_publish_mismatch() { + let server = NatsServer::start().await; + let js = connect(&server).await; + // NATS 2.10 (the testcontainer server version) does not persist + // `allow_atomic_publish`, so it always round-trips as `false` regardless + // of what is requested at creation. Creating with the (honored) default + // of `false` and then requiring `true` still exercises the mismatch + // path without depending on the server actually storing `true`. + let mut config = stream_config("ATOMIC_STREAM", "atomic.stream.>"); + config.allow_atomic_publish = false; + ensure_stream(&js, config.clone()).await.expect("create stream"); + + config.allow_atomic_publish = true; + let error = ensure_stream(&js, config) + .await + .expect_err("allow_atomic_publish mismatch should be rejected"); + + assert!(matches!( + error, + EnsureStreamError::ConfigMismatch(StreamConfigMismatch::AllowAtomicPublish { stream, expected: true, actual: false }) + if stream == "ATOMIC_STREAM" + )); +} + +#[tokio::test] +async fn ensure_stream_rejects_missing_subject() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let config = stream_config("SUBJECT_STREAM", "subject.stream.one"); + ensure_stream(&js, config).await.expect("create stream"); + + let wider = stream_config("SUBJECT_STREAM", "subject.stream.two"); + let error = ensure_stream(&js, wider) + .await + .expect_err("missing subject should be rejected"); + + assert!(matches!( + error, + EnsureStreamError::ConfigMismatch(StreamConfigMismatch::MissingSubject { stream, subject }) + if stream == "SUBJECT_STREAM" && subject == "subject.stream.two" + )); +} + +#[tokio::test] +async fn ensure_stream_is_idempotent_under_concurrent_creation() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let config = stream_config("RACE_STREAM", "race.stream.>"); + + let (first, second) = tokio::join!(ensure_stream(&js, config.clone()), ensure_stream(&js, config)); + + assert_eq!( + first.expect("first racer should succeed").cached_info().config.name, + "RACE_STREAM" + ); + assert_eq!( + second.expect("second racer should succeed").cached_info().config.name, + "RACE_STREAM" + ); +} + +#[tokio::test] +async fn ensure_bucket_creates_a_fresh_bucket() { + let server = NatsServer::start().await; + let js = connect(&server).await; + + let store = ensure_bucket(&js, bucket_config("fresh_bucket", 5)) + .await + .expect("fresh bucket should be created"); + + assert_eq!(store.stream.cached_info().config.max_messages_per_subject, 5); +} + +#[tokio::test] +async fn ensure_bucket_opens_an_existing_matching_bucket() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let config = bucket_config("existing_bucket", 3); + + ensure_bucket(&js, config.clone()) + .await + .expect("first ensure_bucket call should create the bucket"); + + let store = ensure_bucket(&js, config) + .await + .expect("second ensure_bucket call should open the existing bucket"); + + assert_eq!(store.stream.cached_info().config.max_messages_per_subject, 3); +} + +#[tokio::test] +async fn ensure_bucket_rejects_history_mismatch() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let mut config = bucket_config("history_bucket", 2); + ensure_bucket(&js, config.clone()).await.expect("create bucket"); + + config.history = 4; + let error = ensure_bucket(&js, config) + .await + .expect_err("history mismatch should be rejected"); + + assert!(matches!( + error, + EnsureBucketError::ConfigMismatch(KvConfigMismatch::History { bucket, expected: 4, actual: 2 }) + if bucket == "history_bucket" + )); +} + +#[tokio::test] +async fn ensure_bucket_is_idempotent_under_concurrent_creation() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let config = bucket_config("race_bucket", 1); + + let (first, second) = tokio::join!(ensure_bucket(&js, config.clone()), ensure_bucket(&js, config)); + + first.expect("first racer should succeed"); + second.expect("second racer should succeed"); +} diff --git a/rsworkspace/crates/trogon-decider-nats/src/snapshot_store.rs b/rsworkspace/crates/trogon-decider-nats/src/snapshot_store.rs index cc44766b8..bb006bf2f 100644 --- a/rsworkspace/crates/trogon-decider-nats/src/snapshot_store.rs +++ b/rsworkspace/crates/trogon-decider-nats/src/snapshot_store.rs @@ -8,6 +8,9 @@ use async_nats::jetstream::kv; use bytes::Bytes; use futures::StreamExt; +use opentelemetry::global; +use opentelemetry::metrics::Counter; +use std::sync::OnceLock; use std::{borrow::Cow, collections::BTreeMap, convert::Infallible}; use trogon_decider_runtime::StreamPosition; use trogon_decider_runtime::snapshot::{ @@ -18,6 +21,55 @@ use trogon_nats::jetstream::{ JetStreamKeyValueDeleteExpectRevision, JetStreamKeyValueUpdate, JetStreamKvCreate, JetStreamKvEntry, JetStreamKvGet, JetStreamKvKeys, }; +use trogon_semconv::metric; + +const METER_NAME: &str = "trogon-decider-nats"; + +struct SnapshotStoreMetrics { + kv_read_failures: Counter, + kv_write_failures: Counter, +} + +impl SnapshotStoreMetrics { + fn new() -> Self { + let meter = global::meter(METER_NAME); + Self { + kv_read_failures: metric::build_decider_snapshot_kv_read_failures(&meter), + kv_write_failures: metric::build_decider_snapshot_kv_write_failures(&meter), + } + } +} + +static METRICS: OnceLock = OnceLock::new(); + +fn metrics() -> &'static SnapshotStoreMetrics { + METRICS.get_or_init(SnapshotStoreMetrics::new) +} + +fn record_kv_failure(error: &SnapshotKvError) { + match error { + SnapshotKvError::ListSnapshotKeys { .. } + | SnapshotKvError::ReadSnapshotKey { .. } + | SnapshotKvError::ReadSnapshotValue { .. } + | SnapshotKvError::ReadSnapshotEntry { .. } + | SnapshotKvError::ReadCheckpointEntry { .. } + | SnapshotKvError::ReadEntryForUpdate { .. } + | SnapshotKvError::ReadEntryForSnapshotUpdate { .. } + | SnapshotKvError::ReadEntryForDelete { .. } => { + metrics().kv_read_failures.add(1, &[]); + } + SnapshotKvError::AdvanceCheckpoint { .. } + | SnapshotKvError::CreateCheckpoint { .. } + | SnapshotKvError::UpdateEntry { .. } + | SnapshotKvError::CreateEntry { .. } + | SnapshotKvError::UpdateSnapshotEntry { .. } + | SnapshotKvError::CreateSnapshotEntry { .. } + | SnapshotKvError::DeleteEntry { .. } => { + metrics().kv_write_failures.add(1, &[]); + } + SnapshotKvError::DecodeCheckpoint { .. } => {} + } +} /// Bound covering all KV bucket operations used by snapshot helpers. pub trait SnapshotKvBucket: @@ -246,6 +298,7 @@ impl SnapshotChange { impl From for SnapshotStoreError { fn from(source: SnapshotKvError) -> Self { + record_kv_failure(&source); Self::Kv(source) } } diff --git a/rsworkspace/crates/trogon-decider-nats/src/store.rs b/rsworkspace/crates/trogon-decider-nats/src/store.rs index 8851566b3..76bca5a5d 100644 --- a/rsworkspace/crates/trogon-decider-nats/src/store.rs +++ b/rsworkspace/crates/trogon-decider-nats/src/store.rs @@ -1,6 +1,14 @@ use std::convert::Infallible; +#[cfg(not(coverage))] +use std::sync::OnceLock; +#[cfg(not(coverage))] +use std::time::Instant; use async_nats::jetstream::{self, kv}; +#[cfg(not(coverage))] +use opentelemetry::metrics::Histogram; +#[cfg(not(coverage))] +use opentelemetry::{global, metrics::Counter}; #[cfg(any(test, not(coverage)))] use trogon_decider_runtime::ReadFrom; #[cfg(not(coverage))] @@ -14,11 +22,53 @@ use trogon_decider_runtime::{ StreamAppend, StreamRead, }; use trogon_decider_runtime::{StreamPosition, StreamWritePrecondition}; +#[cfg(not(coverage))] +use trogon_semconv::{attribute, metric, span}; use crate::snapshot_store::{NatsSnapshotConfig, SnapshotStoreError}; use crate::stream_store::StreamStoreError; #[cfg(not(coverage))] use crate::stream_store::{StreamSubjectResolver, append_stream as append_subject_stream, read_subject_stream}; +#[cfg(not(coverage))] +use tracing::Instrument; + +#[cfg(not(coverage))] +const METER_NAME: &str = "trogon-decider-nats"; + +#[cfg(not(coverage))] +struct StoreMetrics { + append_duration: Histogram, + append_conflicts: Counter, +} + +#[cfg(not(coverage))] +impl StoreMetrics { + fn new() -> Self { + let meter = global::meter(METER_NAME); + Self { + append_duration: metric::build_decider_append_duration(&meter), + append_conflicts: metric::build_decider_append_conflicts(&meter), + } + } +} + +#[cfg(not(coverage))] +static METRICS: OnceLock = OnceLock::new(); + +#[cfg(not(coverage))] +fn metrics() -> &'static StoreMetrics { + METRICS.get_or_init(StoreMetrics::new) +} + +#[cfg(not(coverage))] +fn write_precondition_attribute(precondition: StreamWritePrecondition) -> attribute::WritePrecondition { + match precondition { + StreamWritePrecondition::Any => attribute::WritePrecondition::Any, + StreamWritePrecondition::StreamExists => attribute::WritePrecondition::StreamExists, + StreamWritePrecondition::NoStream => attribute::WritePrecondition::NoStream, + StreamWritePrecondition::At(_) => attribute::WritePrecondition::At, + } +} #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] /// Optimistic concurrency conflict details for a failed stream append. @@ -230,30 +280,51 @@ where let stream_id = request.stream_id; let expected_state = request.stream_write_precondition; let events = request.events; - let subject_state = self - .subject_resolver - .resolve_subject_state(self.events_stream(), stream_id) - .await - .map_err(JetStreamStoreError::ResolveSubject)?; - let current_position = subject_state.current_position; - let expected_last_subject_sequence = - resolve_expected_last_subject_sequence(stream_id, expected_state, current_position)?; - - let stream_position = append_subject_stream( - self.as_jetstream(), - subject_state.subject, - expected_last_subject_sequence, - &events, - ) + let span = tracing::info_span!( + span::DECIDER_APPEND_STREAM, + otel.kind = "client", + stream_id = %stream_id.as_ref(), + write_precondition = %write_precondition_attribute(expected_state).as_str(), + ); + + async move { + let subject_state = self + .subject_resolver + .resolve_subject_state(self.events_stream(), stream_id) + .await + .map_err(JetStreamStoreError::ResolveSubject)?; + let current_position = subject_state.current_position; + let expected_last_subject_sequence = + resolve_expected_last_subject_sequence(stream_id, expected_state, current_position)?; + + let append_start = Instant::now(); + let append_result = append_subject_stream( + self.as_jetstream(), + subject_state.subject, + expected_last_subject_sequence, + &events, + ) + .await; + metrics() + .append_duration + .record(append_start.elapsed().as_secs_f64(), &[]); + + let stream_position = append_result.map_err(|source| match source { + StreamStoreError::WrongExpectedVersion => { + metrics().append_conflicts.add(1, &[]); + JetStreamStoreError::OptimisticConcurrencyConflict(OptimisticConcurrencyConflictError::new( + stream_id.to_string(), + expected_state, + current_position, + )) + } + other => JetStreamStoreError::AppendStream(other), + })?; + + Ok(AppendStreamResponse { stream_position }) + } + .instrument(span) .await - .map_err(|source| match source { - StreamStoreError::WrongExpectedVersion => JetStreamStoreError::OptimisticConcurrencyConflict( - OptimisticConcurrencyConflictError::new(stream_id.to_string(), expected_state, current_position), - ), - other => JetStreamStoreError::AppendStream(other), - })?; - - Ok(AppendStreamResponse { stream_position }) } } diff --git a/rsworkspace/crates/trogon-decider-nats/src/stream_store.rs b/rsworkspace/crates/trogon-decider-nats/src/stream_store.rs index 05dff3e39..170e74b5a 100644 --- a/rsworkspace/crates/trogon-decider-nats/src/stream_store.rs +++ b/rsworkspace/crates/trogon-decider-nats/src/stream_store.rs @@ -12,7 +12,7 @@ use async_nats::{ context::PublishErrorKind, message::PublishMessage, publish::PublishAck, - stream::{InfoError, LastRawMessageError, RawMessageError}, + stream::{InfoError, LastRawMessageError}, }, subject::ToSubject, }; @@ -21,11 +21,11 @@ use std::{fmt, future::IntoFuture}; use trogon_decider_runtime::{ Event, EventId, FromEntriesError, Headers, InvalidStreamPosition, StreamEvent, StreamPosition, }; -use trogon_nats::jetstream::{ - JetStreamGetRawMessage, JetStreamGetStreamInfo, JetStreamLastRawMessageBySubject, JetStreamPublishMessage, -}; +use trogon_nats::jetstream::{JetStreamLastRawMessageBySubject, JetStreamPublishMessage}; use trogon_std::{NowV7, UuidV7Generator}; +mod replay; + type StreamMessage = async_nats::jetstream::message::StreamMessage; const NATS_BATCH_COMMIT: &str = "Nats-Batch-Commit"; @@ -118,11 +118,6 @@ pub enum ReadStreamError { #[source] source: LastRawMessageError, }, - #[error("failed to read stream message: {source}")] - ReadStreamMessage { - #[source] - source: RawMessageError, - }, #[error("failed to convert stream message timestamp into recorded event time: {subject}")] InvalidMessageTimestamp { subject: String }, #[error("failed to read stream message event id: {source}")] @@ -144,6 +139,28 @@ pub enum ReadStreamError { }, #[error("failed to read stream message event envelope: stream message is missing {header_name} header")] MissingMessageHeader { header_name: &'static str }, + #[error("failed to create replay consumer: {source}")] + CreateReplayConsumer { + #[source] + source: async_nats::jetstream::stream::ConsumerError, + }, + #[error("failed to open replay message stream: {source}")] + OpenReplayMessageStream { + #[source] + source: async_nats::jetstream::consumer::StreamError, + }, + #[error("failed to read replay message: {source}")] + ReadReplayMessage { + #[source] + source: async_nats::jetstream::consumer::pull::OrderedError, + }, + #[error("failed to read replay message delivery metadata: {source}")] + ReadReplayMessageInfo { + #[source] + source: async_nats::Error, + }, + #[error("replay message stream ended before reaching sequence {to_sequence}")] + ReplayEndedBeforeTarget { to_sequence: u64 }, } #[derive(Debug, thiserror::Error)] @@ -324,10 +341,10 @@ fn build_publish_message( } /// Reads all events from the JetStream stream starting at a stream sequence. -pub async fn read_stream(stream: &S, from_sequence: u64) -> Result, StreamStoreError> -where - S: JetStreamGetStreamInfo + JetStreamGetRawMessage, -{ +pub async fn read_stream( + stream: &jetstream::stream::Stream, + from_sequence: u64, +) -> Result, StreamStoreError> { let info = stream .get_info() .await @@ -335,24 +352,21 @@ where read_stream_range(stream, from_sequence, info.state.last_sequence).await } -#[cfg(any(test, not(coverage)))] -pub(crate) async fn read_subject_stream( - stream: &S, +pub(crate) async fn read_subject_stream( + stream: &jetstream::stream::Stream, stream_id: &str, subject: &str, from_sequence: u64, to_sequence: u64, -) -> Result, StreamStoreError> -where - S: JetStreamGetRawMessage, -{ +) -> Result, StreamStoreError> { let stream_id = stream_id.to_string(); - read_message_range( + replay::replay_ordered_range( stream, + Some(subject), from_sequence, to_sequence, - |message| message.subject.as_str() == subject, - |_| stream_id.clone(), + replay::ReplayRetryPolicy::default(), + move |_| stream_id.clone(), ) .await } @@ -386,71 +400,22 @@ where } /// Reads events from an inclusive JetStream stream sequence range. -pub async fn read_stream_range( - stream: &S, +pub async fn read_stream_range( + stream: &jetstream::stream::Stream, from_sequence: u64, to_sequence: u64, -) -> Result, StreamStoreError> -where - S: JetStreamGetRawMessage, -{ - read_message_range( +) -> Result, StreamStoreError> { + replay::replay_ordered_range( stream, + None, from_sequence, to_sequence, - |_| true, + replay::ReplayRetryPolicy::default(), |message| message.subject.to_string(), ) .await } -async fn read_message_range( - stream: &S, - from_sequence: u64, - to_sequence: u64, - mut include: impl FnMut(&StreamMessage) -> bool, - mut stream_id: impl FnMut(&StreamMessage) -> String, -) -> Result, StreamStoreError> -where - S: JetStreamGetRawMessage, -{ - if from_sequence == 0 || to_sequence == 0 || from_sequence > to_sequence { - return Ok(Vec::new()); - } - - let mut events = Vec::new(); - for sequence in from_sequence..=to_sequence { - let Some(message) = read_raw_message(stream, sequence).await? else { - continue; - }; - if !include(&message) { - continue; - } - let stream_id = stream_id(&message); - events.push(record_stream_message(message, stream_id)?); - } - - Ok(events) -} - -async fn read_raw_message(stream: &S, sequence: u64) -> Result, StreamStoreError> -where - S: JetStreamGetRawMessage, -{ - match stream.get_raw_message(sequence).await { - Ok(message) => Ok(Some(message)), - Err(source) - if matches!( - source.kind(), - async_nats::jetstream::stream::RawMessageErrorKind::NoMessageFound - ) => - { - Ok(None) - } - Err(source) => Err(ReadStreamError::ReadStreamMessage { source }.into()), - } -} - /// Converts a raw JetStream message into a decider stream event. /// /// The message must include the NATS message id and Trogon event type headers. diff --git a/rsworkspace/crates/trogon-decider-nats/src/stream_store/replay.rs b/rsworkspace/crates/trogon-decider-nats/src/stream_store/replay.rs new file mode 100644 index 000000000..ccc5b1cdd --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/stream_store/replay.rs @@ -0,0 +1,237 @@ +//! Subject-filtered, sequence-ordered JetStream replay. +//! +//! Reads use an ordered JetStream consumer with a `filter_subject` and a +//! `DeliverPolicy::ByStartSequence` instead of scanning every physical +//! sequence with a direct-get, so only messages that match the requested +//! subject (or none, for a full-stream read) cross the wire. The consumer is +//! self-healing for most disconnects (`async_nats`'s `Ordered` stream +//! recreates itself on missed heartbeats, consumer deletion, and +//! no-responders); the residual failure kinds it does not recover from are +//! retried here by recreating the consumer from the last successfully +//! processed sequence, bounded by [`ReplayRetryPolicy`]. + +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +use async_nats::jetstream; +use async_nats::jetstream::consumer::pull; +use async_nats::jetstream::consumer::{DeliverPolicy, ReplayPolicy, StreamErrorKind}; +use async_nats::jetstream::stream::ConsumerErrorKind; +use futures::StreamExt; +use opentelemetry::global; +use opentelemetry::metrics::{Counter, Histogram}; +use tracing::Instrument; +use trogon_decider_runtime::StreamEvent; +use trogon_semconv::{metric, span}; + +use super::{ReadStreamError, StreamMessage, StreamStoreError, record_stream_message}; + +const METER_NAME: &str = "trogon-decider-nats"; + +struct ReplayMetrics { + replay_duration: Histogram, + replay_retries: Counter, +} + +impl ReplayMetrics { + fn new() -> Self { + let meter = global::meter(METER_NAME); + Self { + replay_duration: metric::build_decider_replay_duration(&meter), + replay_retries: metric::build_decider_replay_retries(&meter), + } + } +} + +static METRICS: OnceLock = OnceLock::new(); + +fn metrics() -> &'static ReplayMetrics { + METRICS.get_or_init(ReplayMetrics::new) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ReplayRetryPolicy { + max_attempts: u32, + base_delay: Duration, + max_delay: Duration, +} + +impl ReplayRetryPolicy { + pub(super) const fn new(max_attempts: u32, base_delay: Duration, max_delay: Duration) -> Self { + Self { + max_attempts, + base_delay, + max_delay, + } + } + + pub(super) fn decide(&self, attempts_used: u32) -> RetryDecision { + if attempts_used >= self.max_attempts { + return RetryDecision::GiveUp; + } + let exponent = attempts_used.min(31); + let multiplier = 1u32.checked_shl(exponent).unwrap_or(u32::MAX); + let delay = self.base_delay.saturating_mul(multiplier).min(self.max_delay); + RetryDecision::Retry { delay } + } +} + +impl Default for ReplayRetryPolicy { + fn default() -> Self { + Self::new(5, Duration::from_millis(100), Duration::from_secs(5)) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RetryDecision { + Retry { delay: Duration }, + GiveUp, +} + +pub(super) fn is_empty_replay_range(from_sequence: u64, to_sequence: u64) -> bool { + from_sequence == 0 || to_sequence == 0 || from_sequence > to_sequence +} + +fn replay_consumer_config(filter_subject: Option<&str>, start_sequence: u64) -> pull::OrderedConfig { + pull::OrderedConfig { + deliver_policy: DeliverPolicy::ByStartSequence { start_sequence }, + replay_policy: ReplayPolicy::Instant, + filter_subject: filter_subject.unwrap_or_default().to_string(), + ..Default::default() + } +} + +pub(super) fn is_transient_replay_error(error: &StreamStoreError) -> bool { + let StreamStoreError::Read(error) = error else { + return false; + }; + match error { + ReadStreamError::CreateReplayConsumer { source } => { + matches!(source.kind(), ConsumerErrorKind::TimedOut | ConsumerErrorKind::Request) + } + ReadStreamError::OpenReplayMessageStream { source } => matches!(source.kind(), StreamErrorKind::TimedOut), + ReadStreamError::ReadReplayMessage { source } => matches!(source.kind(), pull::OrderedErrorKind::Pull), + ReadStreamError::ReplayEndedBeforeTarget { .. } => true, + _ => false, + } +} + +async fn replay_attempt( + stream: &jetstream::stream::Stream, + filter_subject: Option<&str>, + from_sequence: u64, + to_sequence: u64, + stream_id: &mut impl FnMut(&StreamMessage) -> String, + events: &mut Vec, +) -> Result<(), StreamStoreError> { + let consumer = stream + .create_consumer(replay_consumer_config(filter_subject, from_sequence)) + .await + .map_err(|source| ReadStreamError::CreateReplayConsumer { source })?; + let mut messages = consumer + .messages() + .await + .map_err(|source| ReadStreamError::OpenReplayMessageStream { source })?; + + while let Some(message) = messages.next().await { + let message = message.map_err(|source| ReadStreamError::ReadReplayMessage { source })?; + let info = message + .info() + .map_err(|source| ReadStreamError::ReadReplayMessageInfo { source })?; + let sequence = info.stream_sequence; + if sequence > to_sequence { + return Ok(()); + } + + let stream_message = StreamMessage { + subject: message.subject.clone(), + sequence, + headers: message.headers.clone().unwrap_or_default(), + payload: message.payload.clone(), + time: info.published, + }; + let id = stream_id(&stream_message); + events.push(record_stream_message(stream_message, id)?); + + if sequence >= to_sequence { + return Ok(()); + } + } + + Err(ReadStreamError::ReplayEndedBeforeTarget { to_sequence }.into()) +} + +/// Replays events from an ordered, subject-filtered JetStream consumer over +/// the inclusive sequence range `[from_sequence, to_sequence]`. +/// +/// `to_sequence` is a snapshot bound taken by the caller before the replay +/// starts; the replay never reads past it, even if the stream keeps growing +/// while the consumer is open. A transient failure (a reconnect, a timeout, +/// the message stream ending early) recreates the consumer starting from the +/// sequence right after the last event this call already produced, bounded +/// by `retry_policy`. Any other failure propagates immediately. +pub(super) async fn replay_ordered_range( + stream: &jetstream::stream::Stream, + filter_subject: Option<&str>, + from_sequence: u64, + to_sequence: u64, + retry_policy: ReplayRetryPolicy, + mut stream_id: impl FnMut(&StreamMessage) -> String, +) -> Result, StreamStoreError> { + let span = tracing::info_span!( + span::DECIDER_REPLAY_STREAM, + otel.kind = "client", + stream_id = filter_subject.unwrap_or_default(), + ); + let start = Instant::now(); + + let result = async move { + if is_empty_replay_range(from_sequence, to_sequence) { + return Ok(Vec::new()); + } + + let mut events = Vec::new(); + let mut next_sequence = from_sequence; + let mut attempts_used = 0u32; + + loop { + let Err(error) = replay_attempt( + stream, + filter_subject, + next_sequence, + to_sequence, + &mut stream_id, + &mut events, + ) + .await + else { + return Ok(events); + }; + + next_sequence = events + .last() + .map_or(from_sequence, |event| event.stream_position.as_u64().saturating_add(1)); + + if !is_transient_replay_error(&error) { + return Err(error); + } + + match retry_policy.decide(attempts_used) { + RetryDecision::Retry { delay } => { + attempts_used = attempts_used.saturating_add(1); + metrics().replay_retries.add(1, &[]); + tokio::time::sleep(delay).await; + } + RetryDecision::GiveUp => return Err(error), + } + } + } + .instrument(span) + .await; + + metrics().replay_duration.record(start.elapsed().as_secs_f64(), &[]); + result +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-nats/src/stream_store/replay/tests.rs b/rsworkspace/crates/trogon-decider-nats/src/stream_store/replay/tests.rs new file mode 100644 index 000000000..b382e2a51 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-nats/src/stream_store/replay/tests.rs @@ -0,0 +1,142 @@ +use std::time::Duration; + +use async_nats::jetstream::consumer::StreamErrorKind; +use async_nats::jetstream::consumer::pull::OrderedErrorKind; +use async_nats::jetstream::stream::ConsumerErrorKind; + +use super::{ReplayRetryPolicy, RetryDecision, is_empty_replay_range, is_transient_replay_error}; +use crate::stream_store::{ReadStreamError, StreamStoreError}; + +#[test] +fn is_empty_replay_range_rejects_invalid_bounds() { + assert!(is_empty_replay_range(0, 1)); + assert!(is_empty_replay_range(1, 0)); + assert!(is_empty_replay_range(2, 1)); + assert!(!is_empty_replay_range(1, 1)); + assert!(!is_empty_replay_range(1, 3)); +} + +#[test] +fn retry_policy_gives_up_after_max_attempts() { + let policy = ReplayRetryPolicy::new(3, Duration::from_millis(1), Duration::from_secs(1)); + + assert!(matches!(policy.decide(0), RetryDecision::Retry { .. })); + assert!(matches!(policy.decide(1), RetryDecision::Retry { .. })); + assert!(matches!(policy.decide(2), RetryDecision::Retry { .. })); + assert_eq!(policy.decide(3), RetryDecision::GiveUp); + assert_eq!(policy.decide(100), RetryDecision::GiveUp); +} + +#[test] +fn retry_policy_backoff_grows_and_caps() { + let policy = ReplayRetryPolicy::new(10, Duration::from_millis(10), Duration::from_millis(100)); + + assert_eq!( + policy.decide(0), + RetryDecision::Retry { + delay: Duration::from_millis(10) + } + ); + assert_eq!( + policy.decide(1), + RetryDecision::Retry { + delay: Duration::from_millis(20) + } + ); + assert_eq!( + policy.decide(2), + RetryDecision::Retry { + delay: Duration::from_millis(40) + } + ); + assert_eq!( + policy.decide(3), + RetryDecision::Retry { + delay: Duration::from_millis(80) + } + ); + assert_eq!( + policy.decide(4), + RetryDecision::Retry { + delay: Duration::from_millis(100) + } + ); + assert_eq!( + policy.decide(9), + RetryDecision::Retry { + delay: Duration::from_millis(100) + } + ); +} + +#[test] +fn retry_policy_default_is_bounded_and_never_zero_wait() { + let policy = ReplayRetryPolicy::default(); + + let RetryDecision::Retry { delay } = policy.decide(0) else { + panic!("default policy should allow a first retry"); + }; + assert!(delay > Duration::ZERO); +} + +#[test] +fn is_transient_replay_error_classifies_create_consumer_kinds() { + assert!(is_transient_replay_error(&create_consumer_error( + ConsumerErrorKind::TimedOut + ))); + assert!(is_transient_replay_error(&create_consumer_error( + ConsumerErrorKind::Request + ))); + assert!(!is_transient_replay_error(&create_consumer_error( + ConsumerErrorKind::InvalidConsumerType + ))); + assert!(!is_transient_replay_error(&create_consumer_error( + ConsumerErrorKind::InvalidName + ))); + assert!(!is_transient_replay_error(&create_consumer_error( + ConsumerErrorKind::Other + ))); +} + +#[test] +fn is_transient_replay_error_classifies_open_message_stream_kinds() { + assert!(is_transient_replay_error(&open_message_stream_error( + StreamErrorKind::TimedOut + ))); + assert!(!is_transient_replay_error(&open_message_stream_error( + StreamErrorKind::Other + ))); +} + +#[test] +fn is_transient_replay_error_classifies_read_message_kinds() { + assert!(is_transient_replay_error(&read_message_error(OrderedErrorKind::Pull))); + assert!(!is_transient_replay_error(&read_message_error( + OrderedErrorKind::PushBasedConsumer + ))); + assert!(!is_transient_replay_error(&read_message_error(OrderedErrorKind::Other))); +} + +#[test] +fn is_transient_replay_error_treats_replay_ended_before_target_as_transient() { + let error: StreamStoreError = ReadStreamError::ReplayEndedBeforeTarget { to_sequence: 9 }.into(); + + assert!(is_transient_replay_error(&error)); +} + +#[test] +fn is_transient_replay_error_treats_non_read_errors_as_fatal() { + assert!(!is_transient_replay_error(&StreamStoreError::WrongExpectedVersion)); +} + +fn create_consumer_error(kind: ConsumerErrorKind) -> StreamStoreError { + ReadStreamError::CreateReplayConsumer { source: kind.into() }.into() +} + +fn open_message_stream_error(kind: StreamErrorKind) -> StreamStoreError { + ReadStreamError::OpenReplayMessageStream { source: kind.into() }.into() +} + +fn read_message_error(kind: OrderedErrorKind) -> StreamStoreError { + ReadStreamError::ReadReplayMessage { source: kind.into() }.into() +} diff --git a/rsworkspace/crates/trogon-decider-nats/src/stream_store/tests.rs b/rsworkspace/crates/trogon-decider-nats/src/stream_store/tests.rs index 9ebfdeeb0..c6d744094 100644 --- a/rsworkspace/crates/trogon-decider-nats/src/stream_store/tests.rs +++ b/rsworkspace/crates/trogon-decider-nats/src/stream_store/tests.rs @@ -1,20 +1,21 @@ +use std::time::Duration; + use async_nats::{ HeaderMap, header::{NATS_EXPECTED_LAST_SUBJECT_SEQUENCE, NATS_MESSAGE_ID}, - jetstream::{ - context::PublishErrorKind, - message::StreamMessage, - stream::{LastRawMessageError, LastRawMessageErrorKind}, - }, + jetstream, + jetstream::{context::PublishErrorKind, stream::LastRawMessageError, stream::LastRawMessageErrorKind}, }; use bytes::Bytes; -use time::OffsetDateTime; +use testcontainers_modules::nats::{Nats, NatsServerCmd}; +use testcontainers_modules::testcontainers::{ContainerAsync, ImageExt, runners::AsyncRunner}; use trogon_nats::jetstream::JetStreamGetStream; use trogon_nats::jetstream::mocks::{MockJetStreamConsumerFactory, MockJetStreamPublishMessage}; use uuid::Uuid; use trogon_decider_runtime::{Event, EventId, Headers, StreamPosition}; +use super::replay::is_empty_replay_range; use super::{ NATS_BATCH_COMMIT, NATS_BATCH_ID, NATS_BATCH_SEQUENCE, StreamStoreError, StreamSubject, TROGON_EVENT_HEADER_PREFIX, TROGON_EVENT_TYPE, append_stream, build_publish_message, headers_from_nats_headers, read_stream, read_stream_range, @@ -161,14 +162,10 @@ fn build_publish_message_omits_occ_header_without_expected_sequence() { #[test] fn read_stream_range_rejects_empty_ranges() { - assert!(read_stream_range_bounds(0, 1)); - assert!(read_stream_range_bounds(1, 0)); - assert!(read_stream_range_bounds(2, 1)); - assert!(!read_stream_range_bounds(1, 1)); -} - -fn read_stream_range_bounds(from_sequence: u64, to_sequence: u64) -> bool { - from_sequence == 0 || to_sequence == 0 || from_sequence > to_sequence + assert!(is_empty_replay_range(0, 1)); + assert!(is_empty_replay_range(1, 0)); + assert!(is_empty_replay_range(2, 1)); + assert!(!is_empty_replay_range(1, 1)); } fn make_event(id: u128, content: &[u8]) -> Event { @@ -184,51 +181,6 @@ fn make_subject(value: &str) -> StreamSubject { StreamSubject::new(value).expect("subject must be valid") } -fn make_stream_message(subject: &str, sequence: u64, event_id: Uuid) -> StreamMessage { - let mut headers = HeaderMap::new(); - headers.insert(NATS_MESSAGE_ID, event_id.to_string().as_str()); - headers.insert(TROGON_EVENT_TYPE, "test.event.v1"); - StreamMessage { - subject: subject.into(), - sequence, - headers, - payload: Bytes::from_static(b"{}"), - time: OffsetDateTime::UNIX_EPOCH, - } -} - -fn make_stream_info(last_sequence: u64) -> async_nats::jetstream::stream::Info { - serde_json::from_value(serde_json::json!({ - "config": { - "name": "TEST_STREAM", - "subjects": [], - "retention": "limits", - "max_consumers": -1, - "max_msgs": -1, - "max_bytes": -1, - "discard": "old", - "max_age": 0, - "storage": "file", - "num_replicas": 1 - }, - "created": "1970-01-01T00:00:00Z", - "state": { - "messages": 0_u64, - "bytes": 0_u64, - "first_seq": 0_u64, - "first_ts": "1970-01-01T00:00:00Z", - "last_seq": last_sequence, - "last_ts": "1970-01-01T00:00:00Z", - "consumer_count": 0_usize, - "num_subjects": 0_u64 - }, - "cluster": null, - "mirror": null, - "sources": [] - })) - .expect("test stream info must be valid") -} - #[tokio::test] async fn append_stream_publishes_single_event_and_returns_position() { let js = MockJetStreamPublishMessage::new(); @@ -373,165 +325,320 @@ async fn append_stream_maps_ack_error_to_publish_error() { } #[tokio::test] -async fn read_stream_range_skips_no_message_found_gaps() { +async fn subject_current_position_returns_none_when_no_message_found() { let factory = MockJetStreamConsumerFactory::new(); - factory.add_raw_message(1, make_stream_message("test.events", 1, Uuid::from_u128(1))); - // sequence 2 missing → mock returns NoMessageFound by default - factory.add_raw_message(3, make_stream_message("test.events", 3, Uuid::from_u128(3))); let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); - let events = read_stream_range(&stream, 1, 3).await.expect("read should succeed"); + let subject = StreamSubject::new("test.events.backup").unwrap(); + let current = subject_current_position(&stream, &subject).await.unwrap(); - assert_eq!(events.len(), 2); - assert_eq!(events[0].stream_position, StreamPosition::try_new(1).unwrap()); - assert_eq!(events[1].stream_position, StreamPosition::try_new(3).unwrap()); - assert_eq!(factory.raw_message_calls(), vec![1, 2, 3]); + assert_eq!(current, None); } #[tokio::test] -async fn read_subject_stream_filters_by_subject() { +async fn subject_current_position_returns_latest_sequence() { let factory = MockJetStreamConsumerFactory::new(); - factory.add_raw_message(1, make_stream_message("test.events.a", 1, Uuid::from_u128(1))); - factory.add_raw_message(2, make_stream_message("test.events.b", 2, Uuid::from_u128(2))); - factory.add_raw_message(3, make_stream_message("test.events.a", 3, Uuid::from_u128(3))); + factory.add_last_raw_message(async_nats::jetstream::message::StreamMessage { + subject: "test.events.backup".into(), + sequence: 17, + headers: async_nats::HeaderMap::new(), + payload: bytes::Bytes::from_static(b"{}"), + time: time::OffsetDateTime::UNIX_EPOCH, + }); let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); - let events = super::read_subject_stream(&stream, "test.events.a", "test.events.a", 1, 3) - .await - .expect("read should succeed"); + let subject = StreamSubject::new("test.events.backup").unwrap(); + let current = subject_current_position(&stream, &subject).await.unwrap(); - assert_eq!(events.len(), 2); - assert_eq!(events[0].stream_id, "test.events.a"); - assert_eq!(events[1].stream_id, "test.events.a"); + assert_eq!(current, Some(StreamPosition::try_new(17).unwrap())); } #[tokio::test] -async fn read_subject_stream_uses_caller_stream_id() { +async fn subject_current_position_propagates_non_no_message_errors() { let factory = MockJetStreamConsumerFactory::new(); - factory.add_raw_message(1, make_stream_message("test.events.a", 1, Uuid::from_u128(1))); + factory.add_last_raw_message_error(LastRawMessageError::new(LastRawMessageErrorKind::Other)); let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); - let events = super::read_subject_stream(&stream, "a", "test.events.a", 1, 1) + let subject = StreamSubject::new("test.events.backup").unwrap(); + let error = subject_current_position(&stream, &subject).await.unwrap_err(); + + assert!(matches!(error, StreamStoreError::Read(_))); +} + +/// A real, ephemeral single-node NATS server with JetStream enabled, used to +/// exercise the ordered-consumer replay path against actual wire behavior +/// that `trogon-nats`'s mocks cannot represent (delivered-message sequence +/// metadata is only available through a real JetStream consumer). +struct NatsServer { + _container: ContainerAsync, + url: String, +} + +impl NatsServer { + async fn start() -> Self { + let cmd = NatsServerCmd::default().with_jetstream(); + let container = Nats::default() + .with_cmd(&cmd) + .start() + .await + .expect("start NATS testcontainer for JetStream replay tests"); + let host = container.get_host().await.expect("get NATS testcontainer host"); + let port = container + .get_host_port_ipv4(4222) + .await + .expect("get NATS testcontainer port"); + Self { + _container: container, + url: format!("{host}:{port}"), + } + } + + fn url(&self) -> &str { + &self.url + } +} + +async fn connect(server: &NatsServer) -> jetstream::Context { + let client = async_nats::ConnectOptions::new() + .connection_timeout(Duration::from_secs(2)) + .connect(server.url()) .await - .expect("read should succeed"); + .expect("connect to NATS testcontainer"); + jetstream::new(client) +} - assert_eq!(events.len(), 1); - assert_eq!(events[0].stream_id, "a"); +async fn create_events_stream( + js: &jetstream::Context, + name: &str, + subject_wildcard: &str, +) -> jetstream::stream::Stream { + js.create_stream(jetstream::stream::Config { + name: name.to_string(), + subjects: vec![subject_wildcard.to_string()], + allow_atomic_publish: true, + ..Default::default() + }) + .await + .expect("create test events stream") } +async fn publish_noise(js: &jetstream::Context, subject: &str, count: usize) { + for index in 0..count { + js.publish(subject.to_string(), Bytes::from(format!("noise-{index}"))) + .await + .expect("send noise publish") + .await + .expect("ack noise publish"); + } +} + +#[cfg(not(coverage))] #[tokio::test] -async fn read_subject_stream_observes_semantic_publish_mock() { - let js = MockJetStreamPublishMessage::new(); - append_stream( - &js, - make_subject("test.events.alpha"), - Some(0), - &[make_event(1, b"alpha-one")], - ) - .await - .expect("alpha publish should succeed"); - append_stream( - &js, - make_subject("test.events.beta"), - Some(0), - &[make_event(2, b"beta-one")], - ) - .await - .expect("beta publish should succeed"); - append_stream( - &js, - make_subject("test.events.alpha"), - Some(1), - &[make_event(3, b"alpha-two")], - ) - .await - .expect("alpha second publish should succeed"); +async fn read_subject_stream_filters_by_subject_amid_heavy_foreign_traffic() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "FILTER_TEST", "filter.test.>").await; + + let alpha_subject = "filter.test.alpha"; + let noise_subject = "filter.test.noise"; + + let mut alpha_positions = Vec::new(); + for index in 0..3u128 { + publish_noise(&js, noise_subject, 25).await; + let position = append_stream( + &js, + make_subject(alpha_subject), + None, + &[make_event(100 + index, format!("alpha-{index}").as_bytes())], + ) + .await + .expect("publish alpha event"); + alpha_positions.push(position); + } + publish_noise(&js, noise_subject, 25).await; - let events = super::read_subject_stream(&js, "alpha", "test.events.alpha", 1, 3) + let to_sequence = subject_current_position(&stream, &make_subject(alpha_subject)) .await - .expect("read should succeed"); + .expect("read alpha current position") + .expect("alpha subject should have a current position") + .as_u64(); - assert_eq!(events.len(), 2); - assert_eq!(events[0].stream_id, "alpha"); - assert_eq!(events[0].stream_position, StreamPosition::try_new(1).unwrap()); - assert_eq!(events[1].stream_position, StreamPosition::try_new(3).unwrap()); + let events = super::read_subject_stream(&stream, "alpha", alpha_subject, 1, to_sequence) + .await + .expect("replay should succeed"); + + assert_eq!(events.len(), 3); + for (event, position) in events.iter().zip(alpha_positions.iter()) { + assert_eq!(event.stream_id, "alpha"); + assert_eq!(event.stream_position, *position); + } } #[tokio::test] -async fn read_stream_range_returns_empty_for_invalid_bounds() { - let factory = MockJetStreamConsumerFactory::new(); - let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); +async fn read_subject_stream_respects_from_and_to_sequence_bounds() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "BOUNDS_TEST", "bounds.test.>").await; + + let subject = "bounds.test.alpha"; + let mut positions = Vec::new(); + for index in 0..5u128 { + let position = append_stream( + &js, + make_subject(subject), + None, + &[make_event(index, format!("event-{index}").as_bytes())], + ) + .await + .expect("publish event"); + positions.push(position); + } + + let middle = super::read_subject_stream(&stream, "alpha", subject, positions[1].as_u64(), positions[3].as_u64()) + .await + .expect("replay should succeed"); + + assert_eq!(middle.len(), 3); + assert_eq!(middle[0].stream_position, positions[1]); + assert_eq!(middle[2].stream_position, positions[3]); - assert!(read_stream_range(&stream, 0, 1).await.unwrap().is_empty()); - assert!(read_stream_range(&stream, 1, 0).await.unwrap().is_empty()); - assert!(read_stream_range(&stream, 5, 3).await.unwrap().is_empty()); - assert!(factory.raw_message_calls().is_empty()); + let inverted = super::read_subject_stream(&stream, "alpha", subject, positions[3].as_u64(), positions[1].as_u64()) + .await + .expect("inverted range should not error"); + assert!(inverted.is_empty()); } #[tokio::test] -async fn read_stream_range_propagates_non_no_message_errors() { - let factory = MockJetStreamConsumerFactory::new(); - factory.add_raw_message_error(1, LastRawMessageErrorKind::Other); - let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); +async fn read_subject_stream_uses_caller_stream_id() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "STREAM_ID_TEST", "stream.id.test.>").await; - let error = read_stream_range(&stream, 1, 1) + let subject = "stream.id.test.alpha"; + let position = append_stream(&js, make_subject(subject), None, &[make_event(1, b"payload")]) .await - .expect_err("error should propagate"); + .expect("publish event"); - assert!(matches!(error, StreamStoreError::Read(_))); + let events = super::read_subject_stream(&stream, "custom-stream-id", subject, 1, position.as_u64()) + .await + .expect("replay should succeed"); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].stream_id, "custom-stream-id"); } #[tokio::test] -async fn read_stream_uses_info_last_sequence_as_upper_bound() { - let factory = MockJetStreamConsumerFactory::new(); - factory.set_info(make_stream_info(2)); - factory.add_raw_message(1, make_stream_message("test.events", 1, Uuid::from_u128(1))); - factory.add_raw_message(2, make_stream_message("test.events", 2, Uuid::from_u128(2))); - let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); +async fn read_stream_range_reads_all_subjects_in_sequence_order() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "ALL_SUBJECTS_TEST", "all.subjects.test.>").await; + + let alpha = "all.subjects.test.alpha"; + let beta = "all.subjects.test.beta"; + let mut expected_subjects = Vec::new(); + for index in 0..4u128 { + let subject = if index % 2 == 0 { alpha } else { beta }; + append_stream( + &js, + make_subject(subject), + None, + &[make_event(index, format!("event-{index}").as_bytes())], + ) + .await + .expect("publish event"); + expected_subjects.push(subject.to_string()); + } - let events = read_stream(&stream, 1).await.expect("read should succeed"); + let events = read_stream_range(&stream, 1, 4).await.expect("replay should succeed"); - assert_eq!(events.len(), 2); - assert_eq!(factory.raw_message_calls(), vec![1, 2]); + assert_eq!(events.len(), 4); + for (index, event) in events.iter().enumerate() { + assert_eq!( + event.stream_position, + StreamPosition::try_new((index as u64) + 1).unwrap() + ); + assert_eq!(event.stream_id, expected_subjects[index]); + } } #[tokio::test] -async fn subject_current_position_returns_none_when_no_message_found() { - let factory = MockJetStreamConsumerFactory::new(); - let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); +async fn read_stream_reads_up_to_current_stream_tail() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "TAIL_TEST", "tail.test.>").await; + + let subject = "tail.test.alpha"; + for index in 0..3u128 { + append_stream( + &js, + make_subject(subject), + None, + &[make_event(index, format!("event-{index}").as_bytes())], + ) + .await + .expect("publish event"); + } - let subject = StreamSubject::new("test.events.backup").unwrap(); - let current = subject_current_position(&stream, &subject).await.unwrap(); + let events = read_stream(&stream, 1).await.expect("replay should succeed"); - assert_eq!(current, None); + assert_eq!(events.len(), 3); } #[tokio::test] -async fn subject_current_position_returns_latest_sequence() { - let factory = MockJetStreamConsumerFactory::new(); - factory.add_last_raw_message(async_nats::jetstream::message::StreamMessage { - subject: "test.events.backup".into(), - sequence: 17, - headers: async_nats::HeaderMap::new(), - payload: bytes::Bytes::from_static(b"{}"), - time: time::OffsetDateTime::UNIX_EPOCH, - }); - let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); +async fn read_stream_range_propagates_fatal_errors_without_retrying() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "DELETED_STREAM_TEST", "deleted.stream.test.>").await; + js.delete_stream("DELETED_STREAM_TEST") + .await + .expect("delete underlying stream"); - let subject = StreamSubject::new("test.events.backup").unwrap(); - let current = subject_current_position(&stream, &subject).await.unwrap(); + let error = read_stream_range(&stream, 1, 1) + .await + .expect_err("replay against a deleted stream should fail"); - assert_eq!(current, Some(StreamPosition::try_new(17).unwrap())); + assert!(matches!(error, StreamStoreError::Read(_))); } #[tokio::test] -async fn subject_current_position_propagates_non_no_message_errors() { - let factory = MockJetStreamConsumerFactory::new(); - factory.add_last_raw_message_error(LastRawMessageError::new(LastRawMessageErrorKind::Other)); - let stream = JetStreamGetStream::get_stream(&factory, "TEST_STREAM").await.unwrap(); +async fn read_subject_stream_resumes_from_next_sequence_after_partial_progress() { + let server = NatsServer::start().await; + let js = connect(&server).await; + let stream = create_events_stream(&js, "RESUME_TEST", "resume.test.>").await; + + let subject = "resume.test.alpha"; + let noise_subject = "resume.test.noise"; + let mut positions = Vec::new(); + for index in 0..5u128 { + publish_noise(&js, noise_subject, 5).await; + let position = append_stream( + &js, + make_subject(subject), + None, + &[make_event(index, format!("event-{index}").as_bytes())], + ) + .await + .expect("publish event"); + positions.push(position); + } + let to_sequence = positions.last().unwrap().as_u64(); - let subject = StreamSubject::new("test.events.backup").unwrap(); - let error = subject_current_position(&stream, &subject).await.unwrap_err(); + let full = super::read_subject_stream(&stream, "alpha", subject, 1, to_sequence) + .await + .expect("full replay should succeed"); + assert_eq!(full.len(), 5); + + // Simulates what the outer retry loop does after a transient failure: it + // recreates the consumer starting right after the last event it already + // produced, instead of restarting the whole range from scratch. + let checkpoint = full[2].stream_position.as_u64(); + let resumed = super::read_subject_stream(&stream, "alpha", subject, checkpoint + 1, to_sequence) + .await + .expect("resumed replay should succeed"); - assert!(matches!(error, StreamStoreError::Read(_))); + assert_eq!(resumed.len(), 2); + assert_eq!(resumed[0].event.id, full[3].event.id); + assert_eq!(resumed[1].event.id, full[4].event.id); + assert_eq!(resumed[0].stream_position, full[3].stream_position); + assert_eq!(resumed[1].stream_position, full[4].stream_position); } diff --git a/rsworkspace/crates/trogon-decider-runtime/Cargo.toml b/rsworkspace/crates/trogon-decider-runtime/Cargo.toml index a46f5a30c..45928775a 100644 --- a/rsworkspace/crates/trogon-decider-runtime/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-runtime/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/TrogonStack/trogonai" test-support = ["trogon-decider/test-support"] [lints.rust] +missing_docs = "warn" warnings = "deny" unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)'] } @@ -23,11 +24,13 @@ unwrap_used = "deny" thiserror = { workspace = true } chrono = { version = "0.4", features = ["serde"] } futures = { workspace = true } +opentelemetry = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -tokio = { workspace = true, features = ["rt"] } +tokio = { workspace = true, features = ["rt", "sync"] } tracing = { workspace = true } trogon-decider = { version = "0.1.0", path = "../trogon-decider" } +trogon-semconv = { workspace = true } trogon-std = { version = "0.1.0", path = "../trogon-std", features = ["uuid"] } uuid = { workspace = true } diff --git a/rsworkspace/crates/trogon-decider-runtime/README.md b/rsworkspace/crates/trogon-decider-runtime/README.md new file mode 100644 index 000000000..ed7bee3b8 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-runtime/README.md @@ -0,0 +1,19 @@ +# trogon-decider-runtime + +The `-runtime` suffix (per [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md)) marks +execution machinery built on top of the domain crate; this crate owns the storage-neutral +contracts a backend adapter implements (`StreamRead`, `StreamAppend`, `SnapshotRead`, +`SnapshotWrite`) and the runtime boundary that drives one command through them: +`CommandExecution`. `CommandExecution` reads a snapshot (if configured) and stream history, +folds it through `Decider::evolve`, calls `Decider::decide`, and appends the resulting events, +normalizing every phase's failure into one `CommandError`. It also owns snapshotting policy +(`SnapshotPolicy`, `SnapshotFailurePolicy`), event headers (`Headers`, distinct from event +payloads), and stream position semantics (`StreamPosition`, `ReadFrom`). + +This crate has no dependency on any specific storage backend; `trogon-decider-nats` is one +adapter that implements its traits against NATS JetStream, and `#[cfg(feature = +"test-support")]`'s `InMemoryStore` is another used only in tests. + +See `src/lib.rs` for a runnable example, and +[Decider Platform](../../../docs/architecture/decider.md) for the full `CommandExecution` flow, +including snapshot failure recovery and the header/ADR#0013 interaction. diff --git a/rsworkspace/crates/trogon-decider-runtime/src/event/codec/event_decode/tests.rs b/rsworkspace/crates/trogon-decider-runtime/src/event/codec/event_decode/tests.rs deleted file mode 100644 index dd0286fb0..000000000 --- a/rsworkspace/crates/trogon-decider-runtime/src/event/codec/event_decode/tests.rs +++ /dev/null @@ -1,26 +0,0 @@ -use super::*; - -#[test] -fn event_data_new_borrows_event_type_and_payload() { - let payload = [1, 2, 3]; - let event = EventData::new("test.event", &payload); - - assert_eq!(event.event_type, "test.event"); - assert_eq!(event.payload, payload); -} - -#[test] -fn decoded_outcome_exposes_event() { - let outcome = EventDecodeOutcome::Decoded("event"); - - assert_eq!(outcome.as_decoded(), Some(&"event")); - assert_eq!(outcome.into_decoded(), Some("event")); -} - -#[test] -fn skipped_outcome_has_no_event() { - let outcome = EventDecodeOutcome::<&str>::Skipped; - - assert_eq!(outcome.as_decoded(), None); - assert_eq!(outcome.into_decoded(), None); -} diff --git a/rsworkspace/crates/trogon-decider-runtime/src/event/codec/event_payload_error/tests.rs b/rsworkspace/crates/trogon-decider-runtime/src/event/codec/event_payload_error/tests.rs deleted file mode 100644 index 33bc19640..000000000 --- a/rsworkspace/crates/trogon-decider-runtime/src/event/codec/event_payload_error/tests.rs +++ /dev/null @@ -1,35 +0,0 @@ -use std::error::Error as _; - -use super::*; - -#[test] -fn decode_preserves_source_error() { - let error = EventPayloadError::Decode(std::io::Error::other("decode failed")); - - assert_eq!(error.to_string(), "decode failed"); - assert!(error.source().is_some()); -} - -#[test] -fn missing_event_has_no_source() { - let error = EventPayloadError::::MissingEvent; - - assert_eq!(error.to_string(), "event payload is missing its concrete event case"); - assert!(error.source().is_none()); -} - -#[test] -fn unknown_event_type_captures_event_type() { - let error = EventPayloadError::::unknown_event_type("trogonai.scheduler.schedules.v1.Unknown"); - - assert!(matches!( - &error, - EventPayloadError::UnknownEventType { event_type } - if event_type == "trogonai.scheduler.schedules.v1.Unknown" - )); - assert_eq!( - error.to_string(), - "unknown event type 'trogonai.scheduler.schedules.v1.Unknown'" - ); - assert!(error.source().is_none()); -} diff --git a/rsworkspace/crates/trogon-decider-runtime/src/event/mod.rs b/rsworkspace/crates/trogon-decider-runtime/src/event/mod.rs index b458e2643..62b65516b 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/event/mod.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/event/mod.rs @@ -20,6 +20,8 @@ pub use event_identity::EventIdentity; pub use stream_event::StreamEvent; pub use trogon_decider::{EventData, EventDecode, EventDecodeOutcome, EventEncode, EventPayloadError, EventType}; +/// A stored event envelope: a decoded event's stable type name, encoded +/// payload bytes, identity, and out-of-band headers. #[derive(Debug, Clone, PartialEq)] pub struct Event { /// Unique identity assigned to this event occurrence. diff --git a/rsworkspace/crates/trogon-decider-runtime/src/execution.rs b/rsworkspace/crates/trogon-decider-runtime/src/execution.rs index abb8d8bd9..b8b2a8222 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/execution.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/execution.rs @@ -22,9 +22,80 @@ use crate::{ StreamEvent, WritePrecondition, }; use trogon_decider::{DecisionFailure, evaluate_decision}; +use trogon_semconv::{attribute, metric, span}; use trogon_std::{NowV7, UuidV7Generator}; -use std::{borrow::Borrow, future::Future, num::NonZeroU64}; +use opentelemetry::metrics::Counter; +use opentelemetry::{KeyValue, global}; +use tracing::Instrument; + +use std::{ + borrow::Borrow, + future::Future, + num::NonZeroU64, + sync::{ + Arc, OnceLock, + atomic::{AtomicUsize, Ordering}, + }, +}; + +const METER_NAME: &str = "trogon-decider-runtime"; + +/// Counters recorded for command execution's replay, snapshot read, and +/// snapshot write phases. +struct ExecutionMetrics { + replay_events: Counter, + snapshot_reads: Counter, + snapshot_writes: Counter, +} + +impl ExecutionMetrics { + fn new() -> Self { + let meter = global::meter(METER_NAME); + Self { + replay_events: metric::build_decider_replay_events(&meter), + snapshot_reads: metric::build_decider_snapshot_reads(&meter), + snapshot_writes: metric::build_decider_snapshot_writes(&meter), + } + } +} + +static METRICS: OnceLock = OnceLock::new(); + +fn metrics() -> &'static ExecutionMetrics { + METRICS.get_or_init(ExecutionMetrics::new) +} + +fn write_precondition_attribute(precondition: StreamWritePrecondition) -> attribute::WritePrecondition { + match precondition { + StreamWritePrecondition::Any => attribute::WritePrecondition::Any, + StreamWritePrecondition::StreamExists => attribute::WritePrecondition::StreamExists, + StreamWritePrecondition::NoStream => attribute::WritePrecondition::NoStream, + StreamWritePrecondition::At(_) => attribute::WritePrecondition::At, + } +} + +fn execute_command_write_precondition( + configured: Option, +) -> Option { + C::WRITE_PRECONDITION.map(StreamWritePrecondition::from).or(configured) +} + +fn decision_outcome_for_error( + error: &CommandError, +) -> attribute::DecisionOutcome { + match error { + CommandError::Decide(_) => attribute::DecisionOutcome::Rejected, + _ => attribute::DecisionOutcome::Faulted, + } +} + +fn record_snapshot_read_outcome(span: &tracing::Span, outcome: attribute::SnapshotOutcome) { + span.record(attribute::SNAPSHOT_OUTCOME, outcome.as_str()); + metrics() + .snapshot_reads + .add(1, &[KeyValue::new(attribute::SNAPSHOT_OUTCOME, outcome.as_str())]); +} type CommandEventTypeError = <::Event as EventType>::Error; type CommandEventPayloadEncodeError = <::Event as EventEncode>::Error; type CommandEventDecodeError = <::Event as EventDecode>::Error; @@ -43,11 +114,28 @@ type CommandWithSnapshotsResult = /// scheduler keeps that future concrete instead of forcing every runtime adapter /// through a boxed `dyn Future`. pub trait SnapshotTaskScheduler { + /// Schedules `task` to run, without blocking the caller on its completion. fn schedule(&self, task: F) where F: Future + Send + 'static; + + /// Waits for every snapshot write this scheduler has accepted to finish. + /// + /// The default implementation resolves immediately. Schedulers that defer + /// writes to work they do not track have nothing to wait for; schedulers + /// that do track outstanding writes, such as + /// [`DrainableSnapshotTaskScheduler`], override this to await them. + fn drain(&self) -> impl Future + Send { + std::future::ready(()) + } } +/// Schedules snapshot writes on the ambient Tokio runtime without tracking them. +/// +/// This scheduler is fire-and-forget: [`Self::drain`] uses the trait default +/// and resolves immediately, even while writes are still in flight. Hosts that +/// need to await outstanding snapshot writes before teardown should use +/// [`DrainableSnapshotTaskScheduler`] instead. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct TokioSnapshotTaskScheduler; @@ -65,6 +153,66 @@ impl SnapshotTaskScheduler for TokioSnapshotTaskScheduler { } } +#[derive(Debug, Default)] +struct SnapshotTaskTracker { + in_flight: AtomicUsize, + idle: tokio::sync::Notify, +} + +/// Schedules snapshot writes on the ambient Tokio runtime and tracks them so +/// hosts can await outstanding writes before teardown. +/// +/// Every scheduled task increments an in-flight counter and decrements it on +/// completion. [`Self::drain`] waits until that counter reaches zero. Clone +/// this scheduler to share the same in-flight tracking between the executions +/// that schedule writes and the host that drains them; cloning is cheap, it +/// only bumps a reference count. +#[derive(Debug, Clone, Default)] +pub struct DrainableSnapshotTaskScheduler { + tasks: Arc, +} + +impl DrainableSnapshotTaskScheduler { + /// Creates a scheduler with no in-flight snapshot writes. + pub fn new() -> Self { + Self::default() + } +} + +impl SnapshotTaskScheduler for DrainableSnapshotTaskScheduler { + fn schedule(&self, task: F) + where + F: Future + Send + 'static, + { + let Ok(handle) = tokio::runtime::Handle::try_current() else { + tracing::error!("Tokio snapshot task scheduler requires an active Tokio runtime"); + return; + }; + + let tasks = Arc::clone(&self.tasks); + tasks.in_flight.fetch_add(1, Ordering::SeqCst); + drop(handle.spawn(async move { + task.await; + if tasks.in_flight.fetch_sub(1, Ordering::SeqCst) == 1 { + tasks.idle.notify_waiters(); + } + })); + } + + fn drain(&self) -> impl Future + Send { + let tasks = Arc::clone(&self.tasks); + async move { + loop { + let idle = tasks.idle.notified(); + if tasks.in_flight.load(Ordering::SeqCst) == 0 { + break; + } + idle.await; + } + } + } +} + #[cfg(any(test, feature = "test-support"))] /// Runs snapshot tasks to completion before returning. /// @@ -89,12 +237,16 @@ impl SnapshotTaskScheduler for ImmediateSnapshotTaskScheduler { } } +/// Outcome of consulting a [`SnapshotPolicy`] after a successful command execution. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SnapshotDecision { + /// Do not take a snapshot for this execution. Skip, + /// Take a snapshot for this execution. Take, } +/// Context passed to a [`SnapshotPolicy`] so it can decide whether to snapshot. #[derive(Debug, Clone, Copy)] pub struct DecideSnapshot<'a, C: Decider> { /// The command that produced the execution result. @@ -108,29 +260,40 @@ pub struct DecideSnapshot<'a, C: Decider> { /// /// `None` means execution started without a snapshot. pub snapshot_position: Option, + /// State after replaying history and applying the newly decided events. pub state: &'a C::State, + /// Events decided by this execution, already appended to the stream. pub events: &'a Events, /// Number of persisted stream events read after the snapshot position. pub replayed_event_count: u64, } +/// Decides whether a command execution should take a snapshot after it appends events. pub trait SnapshotPolicy { + /// Decides whether to take a snapshot for the given execution context. fn decide_snapshot(&self, context: DecideSnapshot<'_, C>) -> SnapshotDecision; } +/// Associates a [`Decider`] with the [`SnapshotPolicy`] its command executions +/// should use, so callers can build a configured [`Snapshots`] without +/// repeating the policy at every call site. pub trait CommandSnapshotPolicy: Decider where Self::State: SnapshotType, { + /// The snapshot policy this decider's command executions use. type SnapshotPolicy: SnapshotPolicy; + /// The snapshot policy instance this decider's command executions use. const SNAPSHOT_POLICY: Self::SnapshotPolicy; + /// Builds the [`Snapshots`] configuration for this decider from a snapshot store. fn snapshots<'a, S>(snapshot_store: &'a S) -> Snapshots<'a, S, Self::SnapshotPolicy> { Snapshots::new(snapshot_store, Self::SNAPSHOT_POLICY) } } +/// A [`SnapshotPolicy`] that never takes a snapshot. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct NoSnapshot; @@ -140,16 +303,20 @@ impl SnapshotPolicy for NoSnapshot { } } +/// A [`SnapshotPolicy`] that takes a snapshot once at least `frequency` events +/// have been read or appended since the last snapshot. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct FrequencySnapshot { frequency: NonZeroU64, } impl FrequencySnapshot { + /// Creates a policy that snapshots every `frequency` events. pub const fn new(frequency: NonZeroU64) -> Self { Self { frequency } } + /// Returns the configured snapshot frequency. pub const fn frequency(self) -> NonZeroU64 { self.frequency } @@ -165,6 +332,7 @@ impl SnapshotPolicy for FrequencySnapshot { } } +/// Successful outcome of one command execution. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ExecutionResult { /// The stream high-watermark after the command append completed. @@ -244,9 +412,15 @@ pub enum CommandError< ReadAfterOverflow(#[source] ReadAfterOverflow), } +/// Error detail for a loaded snapshot whose recorded position the stream +/// cannot prove happened, for example after the stream was truncated or +/// replaced without clearing its snapshots. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SnapshotAheadOfStream { + /// Position recorded on the loaded snapshot. pub snapshot_position: StreamPosition, + /// Stream high-watermark observed when the snapshot was checked, or + /// `None` if the stream has no current position. pub stream_position: Option, } @@ -267,6 +441,82 @@ impl std::fmt::Display for SnapshotAheadOfStream { } } +/// A recoverable snapshot failure, kept distinct from [`CommandError`] so a +/// [`SnapshotFailurePolicy`] can inspect the failed operation before deciding +/// whether the command should fail or recover. +#[derive(Debug)] +pub enum SnapshotFailure<'a, ReadSnapshotError> { + /// The loaded snapshot claims a position newer than the stream can prove. + AheadOfStream(SnapshotAheadOfStream), + /// Reading the snapshot failed, which also covers a snapshot payload that + /// failed to decode, since the read adapter folds decode failures into + /// its own error type. + ReadFailed(&'a ReadSnapshotError), +} + +/// Context passed to a [`SnapshotFailurePolicy`] when a snapshot failure occurs. +#[derive(Debug)] +pub struct SnapshotFailureContext<'a, C: Decider, ReadSnapshotError> { + /// The command that triggered the failing execution. + pub command: &'a C, + /// The failure the policy must decide how to handle. + pub failure: SnapshotFailure<'a, ReadSnapshotError>, +} + +/// The outcome a [`SnapshotFailurePolicy`] chooses for a [`SnapshotFailure`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SnapshotFailureDecision { + /// Fail the command with the concrete [`CommandError`] for this failure. + Fail, + /// Discard the bad snapshot and replay the command from the beginning of + /// the stream, as if no snapshot had ever been taken. + DiscardAndReplay, +} + +/// Chooses how a [`CommandExecution`] reacts to a snapshot it cannot trust. +/// +/// This mirrors [`SnapshotPolicy`]: it lets callers plug in per-command or +/// per-store logic instead of hard-coding one reaction for every decider. +/// [`FailOnSnapshotFailure`] keeps today's behavior of failing the command; +/// [`DiscardAndReplaySnapshotFailure`] discards the bad snapshot and replays +/// from the beginning of the stream. Custom implementations can choose +/// per [`SnapshotFailure`] kind. +pub trait SnapshotFailurePolicy { + /// Decides how the command execution should react to the given snapshot failure. + fn decide_snapshot_failure( + &self, + context: SnapshotFailureContext<'_, C, ReadSnapshotError>, + ) -> SnapshotFailureDecision; +} + +/// Fails the command on any snapshot failure. This is the default policy and +/// matches the runtime's behavior before [`SnapshotFailurePolicy`] existed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct FailOnSnapshotFailure; + +impl SnapshotFailurePolicy for FailOnSnapshotFailure { + fn decide_snapshot_failure( + &self, + _context: SnapshotFailureContext<'_, C, ReadSnapshotError>, + ) -> SnapshotFailureDecision { + SnapshotFailureDecision::Fail + } +} + +/// Discards a bad snapshot and replays the command from the beginning of the +/// stream instead of failing it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct DiscardAndReplaySnapshotFailure; + +impl SnapshotFailurePolicy for DiscardAndReplaySnapshotFailure { + fn decide_snapshot_failure( + &self, + _context: SnapshotFailureContext<'_, C, ReadSnapshotError>, + ) -> SnapshotFailureDecision { + SnapshotFailureDecision::DiscardAndReplay + } +} + #[derive(Debug, thiserror::Error)] enum ReplayStreamError { #[error("{0}")] @@ -352,61 +602,97 @@ impl< } } +/// Marks a [`CommandExecution`] that has not been configured with a +/// snapshot store, so it always replays from the beginning of the stream. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct WithoutSnapshots; +/// Marks a [`Snapshots`] configuration that has not been given a +/// [`SnapshotTaskScheduler`], so snapshot writes are not scheduled at all. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct WithoutSnapshotTaskScheduler; -pub struct Snapshots<'a, S, P, Spawn = WithoutSnapshotTaskScheduler> { +/// Snapshot store, policy, task scheduler, and failure policy configured for +/// a [`CommandExecution`]. +pub struct Snapshots<'a, S, P, Spawn = WithoutSnapshotTaskScheduler, F = FailOnSnapshotFailure> { snapshot_store: &'a S, policy: P, schedule_snapshot_task: Spawn, + failure_policy: F, } -impl<'a, S, P> Snapshots<'a, S, P, WithoutSnapshotTaskScheduler> { +impl<'a, S, P> Snapshots<'a, S, P, WithoutSnapshotTaskScheduler, FailOnSnapshotFailure> { + /// Configures a snapshot store and policy with no task scheduler and the + /// default [`FailOnSnapshotFailure`] failure policy. pub fn new(snapshot_store: &'a S, policy: P) -> Self { Self { snapshot_store, policy, schedule_snapshot_task: WithoutSnapshotTaskScheduler, + failure_policy: FailOnSnapshotFailure, } } } -impl<'a, S, P, Spawn> Snapshots<'a, S, P, Spawn> { +impl<'a, S, P, Spawn, F> Snapshots<'a, S, P, Spawn, F> { fn schedule_snapshot_tasks_with( self, schedule_snapshot_task: NextSpawn, - ) -> Snapshots<'a, S, P, NextSpawn> { + ) -> Snapshots<'a, S, P, NextSpawn, F> { Snapshots { snapshot_store: self.snapshot_store, policy: self.policy, schedule_snapshot_task, + failure_policy: self.failure_policy, + } + } + + fn with_snapshot_failure_policy(self, failure_policy: NextF) -> Snapshots<'a, S, P, Spawn, NextF> { + Snapshots { + snapshot_store: self.snapshot_store, + policy: self.policy, + schedule_snapshot_task: self.schedule_snapshot_task, + failure_policy, } } } +/// Converts a value into a [`Snapshots`] configuration for a [`Decider`]. +/// +/// Implemented for a plain snapshot store reference (using the decider's +/// [`CommandSnapshotPolicy`]) and for an already-built [`Snapshots`], so +/// [`CommandExecution::with_snapshot`] can accept either. pub trait IntoSnapshots<'a, C>: Sized where C: Decider, { + /// The resulting configuration's snapshot store type. type Store; + /// The resulting configuration's snapshot policy type. type Policy; + /// The resulting configuration's snapshot task scheduler type. type SnapshotTaskScheduler; + /// The resulting configuration's snapshot failure policy type. + type FailurePolicy; - fn into_snapshots(self) -> Snapshots<'a, Self::Store, Self::Policy, Self::SnapshotTaskScheduler>; + /// Builds the [`Snapshots`] configuration. + fn into_snapshots( + self, + ) -> Snapshots<'a, Self::Store, Self::Policy, Self::SnapshotTaskScheduler, Self::FailurePolicy>; } -impl<'a, C, S, P, Spawn> IntoSnapshots<'a, C> for Snapshots<'a, S, P, Spawn> +impl<'a, C, S, P, Spawn, F> IntoSnapshots<'a, C> for Snapshots<'a, S, P, Spawn, F> where C: Decider, { type Store = S; type Policy = P; type SnapshotTaskScheduler = Spawn; + type FailurePolicy = F; - fn into_snapshots(self) -> Snapshots<'a, Self::Store, Self::Policy, Self::SnapshotTaskScheduler> { + fn into_snapshots( + self, + ) -> Snapshots<'a, Self::Store, Self::Policy, Self::SnapshotTaskScheduler, Self::FailurePolicy> { self } } @@ -419,12 +705,23 @@ where type Store = S; type Policy = C::SnapshotPolicy; type SnapshotTaskScheduler = WithoutSnapshotTaskScheduler; + type FailurePolicy = FailOnSnapshotFailure; - fn into_snapshots(self) -> Snapshots<'a, Self::Store, Self::Policy, Self::SnapshotTaskScheduler> { + fn into_snapshots( + self, + ) -> Snapshots<'a, Self::Store, Self::Policy, Self::SnapshotTaskScheduler, Self::FailurePolicy> { C::snapshots(self) } } +/// Runtime boundary that applies one [`Decider`] command to one stream: reads +/// the stream (and optionally a snapshot), replays state, decides the next +/// events, and appends them with the configured write precondition. +/// +/// Build one with [`CommandExecution::new`] and configure it with the +/// builder methods before calling `execute`, which is available when `S` is +/// [`WithoutSnapshots`] or a [`Snapshots`] configuration whose store and +/// policy satisfy the trait bounds that method requires. pub struct CommandExecution<'a, E, C, S, G> { event_store: &'a E, command: &'a C, @@ -438,6 +735,9 @@ impl<'a, E, C> CommandExecution<'a, E, C, WithoutSnapshots, UuidV7Generator> where C: Decider, { + /// Starts building an execution for `command` against `event_store`, with + /// no snapshot store, no explicit write precondition, no headers, and the + /// default UUIDv7 event id generator. pub fn new(event_store: &'a E, command: &'a C) -> Self { Self { event_store, @@ -454,6 +754,9 @@ impl<'a, E, C, G> CommandExecution<'a, E, C, WithoutSnapshots, G> where C: Decider, { + /// Configures this execution to read and write snapshots, accepting + /// either a bare snapshot store (using the decider's + /// [`CommandSnapshotPolicy`]) or an explicitly built [`Snapshots`]. #[allow(clippy::type_complexity)] pub fn with_snapshot( self, @@ -467,6 +770,7 @@ where >::Store, >::Policy, >::SnapshotTaskScheduler, + >::FailurePolicy, >, G, > @@ -485,6 +789,10 @@ where } impl<'a, E, C, S, G> CommandExecution<'a, E, C, S, G> { + /// Overrides the stream write precondition used for the append. + /// + /// Ignored when the decider declares its own [`WritePrecondition`] via + /// `C::WRITE_PRECONDITION`; that always takes precedence over this value. pub fn with_write_precondition(mut self, write_precondition: W) -> Self where W: Into>, @@ -493,11 +801,14 @@ impl<'a, E, C, S, G> CommandExecution<'a, E, C, S, G> { self } + /// Sets metadata headers attached to every event this execution appends. pub fn with_headers(mut self, headers: Headers) -> Self { self.headers = headers; self } + /// Replaces the generator used to assign ids to newly decided events that + /// don't already carry one. pub fn with_event_id_generator(self, event_id_generator: NextG) -> CommandExecution<'a, E, C, S, NextG> where NextG: NowV7, @@ -572,11 +883,12 @@ impl<'a, E, C, S, G> CommandExecution<'a, E, C, S, G> { } } -impl<'a, E, S, C, P, Spawn, G> CommandExecution<'a, E, C, Snapshots<'a, S, P, Spawn>, G> { +impl<'a, E, S, C, P, Spawn, F, G> CommandExecution<'a, E, C, Snapshots<'a, S, P, Spawn, F>, G> { + /// Replaces the scheduler used to run snapshot write tasks. pub fn with_task_runtime( self, schedule_snapshot_task: NextSpawn, - ) -> CommandExecution<'a, E, C, Snapshots<'a, S, P, NextSpawn>, G> { + ) -> CommandExecution<'a, E, C, Snapshots<'a, S, P, NextSpawn, F>, G> { CommandExecution { event_store: self.event_store, command: self.command, @@ -586,6 +898,25 @@ impl<'a, E, S, C, P, Spawn, G> CommandExecution<'a, E, C, Snapshots<'a, S, P, Sp event_id_generator: self.event_id_generator, } } + + /// Sets how this execution reacts to a snapshot it cannot trust. + /// + /// Defaults to [`FailOnSnapshotFailure`], which fails the command exactly + /// as before this policy existed. Use [`DiscardAndReplaySnapshotFailure`] + /// or a custom [`SnapshotFailurePolicy`] to recover instead. + pub fn with_snapshot_failure_policy( + self, + failure_policy: NextF, + ) -> CommandExecution<'a, E, C, Snapshots<'a, S, P, Spawn, NextF>, G> { + CommandExecution { + event_store: self.event_store, + command: self.command, + write_precondition: self.write_precondition, + snapshots: self.snapshots.with_snapshot_failure_policy(failure_policy), + headers: self.headers, + event_id_generator: self.event_id_generator, + } + } } impl CommandExecution<'_, E, C, WithoutSnapshots, G> @@ -599,8 +930,35 @@ where CommandEventPayloadEncodeError: std::error::Error + Send + Sync + 'static, CommandEventDecodeError: std::error::Error + Send + Sync + 'static, { + /// Replays the stream from the beginning, decides the command, and + /// appends the resulting events. No snapshot is read or written. pub async fn execute(self) -> CommandWithoutSnapshotsResult { + let write_precondition = execute_command_write_precondition::(self.write_precondition); + let span = tracing::info_span!( + span::DECIDER_EXECUTE_COMMAND, + stream_id = tracing::field::Empty, + write_precondition = tracing::field::Empty, + decision_outcome = tracing::field::Empty, + ); + if let Some(write_precondition) = write_precondition { + span.record( + attribute::WRITE_PRECONDITION, + write_precondition_attribute(write_precondition).as_str(), + ); + } + + let result = self.execute_inner().instrument(span.clone()).await; + let decision_outcome = match &result { + Ok(_) => attribute::DecisionOutcome::Decided, + Err(error) => decision_outcome_for_error(error), + }; + span.record(attribute::DECISION_OUTCOME, decision_outcome.as_str()); + result + } + + async fn execute_inner(self) -> CommandWithoutSnapshotsResult { let stream_id = self.command.stream_id(); + tracing::Span::current().record(attribute::STREAM_ID, stream_id.as_ref()); if has_no_stream_write_precondition::() { let (append_outcome, events, state) = self.append_decision(None, stream_id, C::initial_state()).await?; @@ -620,6 +978,7 @@ where .await .map_err(CommandError::ReadStream)?; let current_position = stream_read.current_position; + metrics().replay_events.add(stream_read.events.len() as u64, &[]); let state = evolve_state_from_stream_events::(C::initial_state(), &stream_read.events)?; let (append_outcome, events, state) = self.append_decision(current_position, stream_id, state).await?; @@ -631,7 +990,7 @@ where } } -impl CommandExecution<'_, E, C, Snapshots<'_, S, P, Spawn>, G> +impl CommandExecution<'_, E, C, Snapshots<'_, S, P, Spawn, F>, G> where C: Decider, C::State: Clone + Send + 'static, @@ -642,6 +1001,7 @@ where S: Clone + SnapshotRead + SnapshotWrite + 'static, P: SnapshotPolicy, Spawn: SnapshotTaskScheduler + Send + Sync, + F: SnapshotFailurePolicy>, G: NowV7, CommandWriteSnapshotError: std::fmt::Display + Send + 'static, CommandEventTypeError: std::error::Error + Send + Sync + 'static, @@ -649,8 +1009,38 @@ where CommandEventDecodeError: std::error::Error + Send + Sync + 'static, C::State: SnapshotType, { + /// Loads a snapshot if one exists, replays only the stream events after + /// it, decides the command, appends the resulting events, and schedules a + /// new snapshot write if the configured [`SnapshotPolicy`] decides to + /// take one. A snapshot the [`SnapshotFailurePolicy`] cannot trust is + /// discarded and replayed from the beginning of the stream instead. pub async fn execute(self) -> CommandWithSnapshotsResult { + let write_precondition = execute_command_write_precondition::(self.write_precondition); + let span = tracing::info_span!( + span::DECIDER_EXECUTE_COMMAND, + stream_id = tracing::field::Empty, + write_precondition = tracing::field::Empty, + decision_outcome = tracing::field::Empty, + ); + if let Some(write_precondition) = write_precondition { + span.record( + attribute::WRITE_PRECONDITION, + write_precondition_attribute(write_precondition).as_str(), + ); + } + + let result = self.execute_inner().instrument(span.clone()).await; + let decision_outcome = match &result { + Ok(_) => attribute::DecisionOutcome::Decided, + Err(error) => decision_outcome_for_error(error), + }; + span.record(attribute::DECISION_OUTCOME, decision_outcome.as_str()); + result + } + + async fn execute_inner(self) -> CommandWithSnapshotsResult { let stream_id = self.command.stream_id(); + tracing::Span::current().record(attribute::STREAM_ID, stream_id.as_ref()); if has_no_stream_write_precondition::() { let (append_outcome, events, state) = self.append_decision(None, stream_id, C::initial_state()).await?; @@ -674,17 +1064,59 @@ where }); } - let snapshot = self + let stream_id_display = stream_id.as_ref().to_string(); + let read_snapshot_span = tracing::info_span!( + span::DECIDER_READ_SNAPSHOT, + stream_id = %stream_id_display, + snapshot_outcome = tracing::field::Empty, + ); + + let mut discarded_bad_snapshot = false; + let (mut snapshot_position, mut state, mut snapshot_outcome) = match self .snapshots .snapshot_store .read_snapshot(ReadSnapshotRequest { snapshot_id: stream_id }) + .instrument(read_snapshot_span.clone()) .await - .map_err(CommandError::ReadSnapshot)?; - let snapshot = snapshot.snapshot; - let snapshot_position = snapshot.as_ref().map(|snapshot| snapshot.position); - let state = snapshot - .map(|snapshot| snapshot.payload) - .unwrap_or_else(C::initial_state); + { + Ok(response) => { + let snapshot = response.snapshot; + let snapshot_position = snapshot.as_ref().map(|snapshot| snapshot.position); + let outcome = if snapshot_position.is_some() { + attribute::SnapshotOutcome::Hit + } else { + attribute::SnapshotOutcome::Miss + }; + let state = snapshot + .map(|snapshot| snapshot.payload) + .unwrap_or_else(C::initial_state); + (snapshot_position, state, outcome) + } + Err(error) => { + let decision = self + .snapshots + .failure_policy + .decide_snapshot_failure(SnapshotFailureContext { + command: self.command, + failure: SnapshotFailure::ReadFailed(&error), + }); + match decision { + SnapshotFailureDecision::Fail => { + record_snapshot_read_outcome(&read_snapshot_span, attribute::SnapshotOutcome::Failed); + return Err(CommandError::ReadSnapshot(error)); + } + SnapshotFailureDecision::DiscardAndReplay => { + discarded_bad_snapshot = true; + ( + None, + C::initial_state(), + attribute::SnapshotOutcome::DiscardedReadFailure, + ) + } + } + } + }; + let from = match snapshot_position { Some(position) => ReadFrom::after(position).map_err(CommandError::ReadAfterOverflow)?, None => ReadFrom::Beginning, @@ -694,29 +1126,78 @@ where .read_stream(ReadStreamRequest { stream_id, from }) .await .map_err(CommandError::ReadStream)?; - let current_position = stream_read.current_position; + let mut current_position = stream_read.current_position; + let mut stream_events = stream_read.events; - if let Some(snapshot_position) = snapshot_position { - ensure_snapshot_not_ahead(snapshot_position, current_position) - .map_err(CommandError::SnapshotAheadOfStream)?; + if let Some(position) = snapshot_position + && let Err(ahead_of_stream) = ensure_snapshot_not_ahead(position, current_position) + { + let decision = self + .snapshots + .failure_policy + .decide_snapshot_failure(SnapshotFailureContext { + command: self.command, + failure: SnapshotFailure::AheadOfStream(ahead_of_stream), + }); + match decision { + SnapshotFailureDecision::Fail => { + record_snapshot_read_outcome(&read_snapshot_span, attribute::SnapshotOutcome::Failed); + return Err(CommandError::SnapshotAheadOfStream(ahead_of_stream)); + } + SnapshotFailureDecision::DiscardAndReplay => { + discarded_bad_snapshot = true; + snapshot_position = None; + state = C::initial_state(); + snapshot_outcome = attribute::SnapshotOutcome::DiscardedAheadOfStream; + + let replay = self + .event_store + .read_stream(ReadStreamRequest { + stream_id, + from: ReadFrom::Beginning, + }) + .await + .map_err(CommandError::ReadStream)?; + current_position = replay.current_position; + stream_events = replay.events; + } + } } - let state = evolve_state_from_stream_events::(state, &stream_read.events)?; - let (append_outcome, events, state) = self.append_decision(current_position, stream_id, state).await?; - let replayed_event_count = stream_read.events.len() as u64; + record_snapshot_read_outcome(&read_snapshot_span, snapshot_outcome); - maybe_take_snapshot( - &self.snapshots, - stream_id, - DecideSnapshot { - command: self.command, - stream_position: append_outcome.stream_position, - snapshot_position, - state: &state, - events: &events, - replayed_event_count, - }, - ); + let state = evolve_state_from_stream_events::(state, &stream_events)?; + let (append_outcome, events, state) = self.append_decision(current_position, stream_id, state).await?; + let replayed_event_count = stream_events.len() as u64; + metrics().replay_events.add(replayed_event_count, &[]); + + if discarded_bad_snapshot { + // The discarded snapshot is still sitting in the store at this + // stream id. The normal policy might choose to skip a snapshot + // this time, which would leave that stale or undecodable payload + // in place for the next execution to trip over again. Writing + // unconditionally here overwrites it with a snapshot the recovered + // execution can trust. + schedule_snapshot_write( + &self.snapshots.schedule_snapshot_task, + self.snapshots.snapshot_store, + stream_id, + Snapshot::new(append_outcome.stream_position, state.clone()), + ); + } else { + maybe_take_snapshot( + &self.snapshots, + stream_id, + DecideSnapshot { + command: self.command, + stream_position: append_outcome.stream_position, + snapshot_position, + state: &state, + events: &events, + replayed_event_count, + }, + ); + } Ok(ExecutionResult { stream_position: append_outcome.stream_position, @@ -753,8 +1234,8 @@ fn ensure_snapshot_not_ahead( } } -fn maybe_take_snapshot( - snapshots: &Snapshots<'_, S, P, Spawn>, +fn maybe_take_snapshot( + snapshots: &Snapshots<'_, S, P, Spawn, F>, stream_id: &C::StreamId, context: DecideSnapshot<'_, C>, ) where @@ -799,13 +1280,24 @@ fn schedule_snapshot_write( let stream_id = stream_id.to_owned(); schedule_snapshot_task.schedule(async move { - if let Err(source) = snapshot_store + let span = tracing::info_span!( + span::DECIDER_WRITE_SNAPSHOT, + stream_id = %stream_id_for_log, + snapshot_write_success = tracing::field::Empty, + ); + let result = snapshot_store .write_snapshot(WriteSnapshotRequest { snapshot_id: stream_id.borrow(), snapshot, }) - .await - { + .instrument(span.clone()) + .await; + let success = result.is_ok(); + span.record(attribute::SNAPSHOT_WRITE_SUCCESS, success); + metrics() + .snapshot_writes + .add(1, &[KeyValue::new(attribute::SNAPSHOT_WRITE_SUCCESS, success)]); + if let Err(source) = result { tracing::warn!(stream_id = %stream_id_for_log, error = %source, "failed to write snapshot"); } }); diff --git a/rsworkspace/crates/trogon-decider-runtime/src/execution/tests.rs b/rsworkspace/crates/trogon-decider-runtime/src/execution/tests.rs index 9d4796039..e4265f540 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/execution/tests.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/execution/tests.rs @@ -563,6 +563,70 @@ async fn tokio_snapshot_task_scheduler_spawns_on_current_runtime() { assert!(executed.load(Ordering::SeqCst)); } +#[tokio::test] +async fn tokio_snapshot_task_scheduler_drain_does_not_wait_for_scheduled_tasks() { + let scheduler = TokioSnapshotTaskScheduler; + let release = Arc::new(tokio::sync::Notify::new()); + let task_release = Arc::clone(&release); + let executed = Arc::new(AtomicBool::new(false)); + let task_executed = Arc::clone(&executed); + + scheduler.schedule(async move { + task_release.notified().await; + task_executed.store(true, Ordering::SeqCst); + }); + + scheduler.drain().await; + + assert!(!executed.load(Ordering::SeqCst)); + + release.notify_one(); + tokio::task::yield_now().await; + + assert!(executed.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn drainable_snapshot_task_scheduler_drain_resolves_immediately_when_idle() { + let scheduler = DrainableSnapshotTaskScheduler::new(); + + scheduler.drain().await; +} + +#[tokio::test] +async fn drainable_snapshot_task_scheduler_drain_awaits_outstanding_task() { + let scheduler = DrainableSnapshotTaskScheduler::new(); + let executed = Arc::new(AtomicBool::new(false)); + let task_executed = Arc::clone(&executed); + + scheduler.schedule(async move { + tokio::task::yield_now().await; + tokio::task::yield_now().await; + task_executed.store(true, Ordering::SeqCst); + }); + + scheduler.drain().await; + + assert!(executed.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn drainable_snapshot_task_scheduler_shares_tracking_across_clones() { + let scheduler = DrainableSnapshotTaskScheduler::new(); + let scheduling_clone = scheduler.clone(); + let executed = Arc::new(AtomicBool::new(false)); + let task_executed = Arc::clone(&executed); + + scheduling_clone.schedule(async move { + tokio::task::yield_now().await; + task_executed.store(true, Ordering::SeqCst); + }); + + scheduler.drain().await; + + assert!(executed.load(Ordering::SeqCst)); +} + #[test] fn immediate_snapshot_task_scheduler_catches_task_panic() { ImmediateSnapshotTaskScheduler.schedule(async { @@ -886,6 +950,123 @@ fn errors_when_snapshot_exists_without_stream_history() { )); } +#[test] +fn fail_on_snapshot_failure_policy_still_errors_on_ahead_of_stream() { + let runtime = FakeRuntime { + snapshot: Some(Snapshot::new(position(3), TestState::Present { enabled: true })), + current_position: Some(position(2)), + ..Default::default() + }; + let command = TestCommand::new("alpha", TestAction::Remove); + + let error = block_on( + CommandExecution::new(&runtime, &command) + .with_snapshot(test_snapshots(&runtime, NoSnapshot)) + .with_snapshot_failure_policy(FailOnSnapshotFailure) + .execute(), + ) + .unwrap_err(); + + assert!(matches!( + error, + CommandError::SnapshotAheadOfStream(SnapshotAheadOfStream { + snapshot_position, + stream_position: Some(stream_position), + }) if snapshot_position == position(3) && stream_position == position(2) + )); + assert!(runtime.written_snapshots.lock().unwrap().is_empty()); +} + +#[test] +fn fail_on_snapshot_failure_policy_still_errors_on_read_failure() { + let runtime = FakeRuntime { + fail_read_snapshot: true, + ..Default::default() + }; + let command = TestCommand::new("alpha", TestAction::Register); + + let error = block_on( + CommandExecution::new(&runtime, &command) + .with_snapshot(test_snapshots(&runtime, NoSnapshot)) + .with_snapshot_failure_policy(FailOnSnapshotFailure) + .execute(), + ) + .unwrap_err(); + + assert!(matches!( + error, + CommandError::ReadSnapshot(TestInfraError::ReadSnapshot) + )); + assert!(runtime.written_snapshots.lock().unwrap().is_empty()); +} + +#[test] +fn discard_and_replay_recovers_from_snapshot_ahead_of_stream() { + let runtime = FakeRuntime { + snapshot: Some(Snapshot::new(position(3), TestState::Present { enabled: true })), + current_position: Some(position(2)), + stream_events: vec![stream_event( + 1, + TestEvent::Registered { + id: "alpha".to_string(), + }, + )], + stream_position: position(3), + ..Default::default() + }; + let command = TestCommand::new("alpha", TestAction::Remove); + + let result = block_on( + CommandExecution::new(&runtime, &command) + .with_snapshot(test_snapshots(&runtime, NoSnapshot)) + .with_snapshot_failure_policy(DiscardAndReplaySnapshotFailure) + .execute(), + ) + .unwrap(); + + assert_eq!(result.state, TestState::Missing); + assert_eq!( + runtime.reads_from.lock().unwrap().as_slice(), + &[ReadFrom::Position(position(4)), ReadFrom::Beginning] + ); + assert_eq!( + runtime.written_snapshots.lock().unwrap().as_slice(), + &[Snapshot::new(position(3), TestState::Missing)] + ); +} + +#[test] +fn discard_and_replay_recovers_from_snapshot_read_failure() { + let runtime = FakeRuntime { + fail_read_snapshot: true, + current_position: Some(position(1)), + stream_events: vec![stream_event( + 1, + TestEvent::Registered { + id: "alpha".to_string(), + }, + )], + stream_position: position(2), + ..Default::default() + }; + let command = TestCommand::new("alpha", TestAction::Disable); + + let result = block_on( + CommandExecution::new(&runtime, &command) + .with_snapshot(test_snapshots(&runtime, NoSnapshot)) + .with_snapshot_failure_policy(DiscardAndReplaySnapshotFailure) + .execute(), + ) + .unwrap(); + + assert_eq!(result.state, TestState::Present { enabled: false }); + assert_eq!(runtime.reads_from.lock().unwrap().as_slice(), &[ReadFrom::Beginning]); + assert_eq!( + runtime.written_snapshots.lock().unwrap().as_slice(), + &[Snapshot::new(position(2), TestState::Present { enabled: false })] + ); +} + #[test] fn propagates_replay_evolve_failures() { let runtime = FakeRuntime { diff --git a/rsworkspace/crates/trogon-decider-runtime/src/headers/from_entries_error.rs b/rsworkspace/crates/trogon-decider-runtime/src/headers/from_entries_error.rs index 0db112e51..c50ce89f9 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/headers/from_entries_error.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/headers/from_entries_error.rs @@ -6,14 +6,18 @@ pub enum FromEntriesError { /// A raw header name failed validation. #[error("event header name '{name}' is not valid")] InvalidName { + /// The raw name that failed validation. name: String, + /// The underlying validation failure. #[source] source: HeaderNameError, }, /// A raw header value failed validation. #[error("event header value for '{name}' is not valid")] InvalidValue { + /// The name of the header whose value failed validation. name: String, + /// The underlying validation failure. #[source] source: HeaderValueError, }, diff --git a/rsworkspace/crates/trogon-decider-runtime/src/lib.rs b/rsworkspace/crates/trogon-decider-runtime/src/lib.rs index bdc2bab39..64666079d 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/lib.rs @@ -73,6 +73,9 @@ pub mod event; pub mod execution; /// Metadata header value objects carried alongside event payloads. pub mod headers; +/// In-memory test double for stream and snapshot storage contracts. +#[cfg(feature = "test-support")] +pub mod memory; /// Snapshot read/write contracts and payload codec traits. pub mod snapshot; /// Stream read/write contracts shared by event store backends. @@ -82,11 +85,15 @@ pub use event::{Event, EventId, EventIdentity, StreamEvent}; #[cfg(any(test, feature = "test-support"))] pub use execution::ImmediateSnapshotTaskScheduler; pub use execution::{ - CommandError, CommandExecution, CommandResult, CommandSnapshotPolicy, DecideSnapshot, ExecutionResult, - FrequencySnapshot, NoSnapshot, SnapshotAheadOfStream, SnapshotDecision, SnapshotPolicy, SnapshotTaskScheduler, - Snapshots, TokioSnapshotTaskScheduler, WithoutSnapshotTaskScheduler, WithoutSnapshots, + CommandError, CommandExecution, CommandResult, CommandSnapshotPolicy, DecideSnapshot, + DiscardAndReplaySnapshotFailure, DrainableSnapshotTaskScheduler, ExecutionResult, FailOnSnapshotFailure, + FrequencySnapshot, NoSnapshot, SnapshotAheadOfStream, SnapshotDecision, SnapshotFailure, SnapshotFailureContext, + SnapshotFailureDecision, SnapshotFailurePolicy, SnapshotPolicy, SnapshotTaskScheduler, Snapshots, + TokioSnapshotTaskScheduler, WithoutSnapshotTaskScheduler, WithoutSnapshots, }; pub use headers::{FromEntriesError, HeaderName, HeaderNameError, HeaderValue, HeaderValueError, Headers}; +#[cfg(feature = "test-support")] +pub use memory::{InMemoryStore, StreamAppendError}; pub use snapshot::{ InvalidSnapshotTypeName, ReadSnapshotRequest, ReadSnapshotResponse, Snapshot, SnapshotPayloadData, SnapshotPayloadDecode, SnapshotPayloadEncode, SnapshotRead, SnapshotType, SnapshotTypeName, SnapshotWrite, diff --git a/rsworkspace/crates/trogon-decider-runtime/src/memory.rs b/rsworkspace/crates/trogon-decider-runtime/src/memory.rs new file mode 100644 index 000000000..1efba02a2 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-runtime/src/memory.rs @@ -0,0 +1,236 @@ +//! In-memory store test double for decider stream and snapshot contracts. +//! +//! [`InMemoryStore`] implements [`StreamRead`], [`StreamAppend`], +//! [`SnapshotRead`], and [`SnapshotWrite`] entirely in process memory. It +//! exists so tests can exercise a [`CommandExecution`](crate::CommandExecution) +//! or a projection against real optimistic concurrency semantics without +//! standing up a backing store such as NATS JetStream or Postgres. +//! +//! A stream's current position is the number of events appended to it, so +//! `StreamWritePrecondition::Any`, `StreamWritePrecondition::NoStream`, +//! `StreamWritePrecondition::StreamExists`, and `StreamWritePrecondition::At` +//! are all enforced against that count before an append is accepted, the same +//! way a storage-backed adapter enforces them against a subject sequence or a +//! row version. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use chrono::Utc; + +use crate::snapshot::{ + EncodedSnapshot, ReadSnapshotRequest, ReadSnapshotResponse, SnapshotDecodeError, SnapshotEncodeError, + SnapshotPayloadDecode, SnapshotPayloadEncode, SnapshotRead, SnapshotType, SnapshotWrite, WriteSnapshotRequest, + WriteSnapshotResponse, decode_snapshot, encode_snapshot, +}; +use crate::stream::{ + AppendStreamRequest, AppendStreamResponse, ReadFrom, ReadStreamRequest, ReadStreamResponse, StreamAppend, + StreamPosition, StreamRead, StreamWritePrecondition, +}; +use crate::{Event, StreamEvent}; + +/// In-memory implementation of [`StreamRead`], [`StreamAppend`], +/// [`SnapshotRead`], and [`SnapshotWrite`] for tests. +/// +/// Cloning an [`InMemoryStore`] returns a handle to the same underlying +/// state; create one store per test fixture with [`InMemoryStore::new`] and +/// clone the handle to share it across concurrent readers and writers. +#[derive(Clone, Default)] +pub struct InMemoryStore { + streams: Arc>>>, + snapshots: Arc>>, +} + +impl InMemoryStore { + /// Creates an empty store. + pub fn new() -> Self { + Self::default() + } +} + +/// Error returned when an append's [`StreamWritePrecondition`] is not +/// satisfied by an [`InMemoryStore`] stream's current state. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum StreamAppendError { + /// The stream's current position did not satisfy the write precondition. + #[error( + "stream '{stream_id}' write precondition violated: expected {expected:?}, {}", + describe_current_position(current_position) + )] + WriteConflict { + /// Stream identity that failed its write precondition. + stream_id: String, + /// Precondition supplied by the caller. + expected: StreamWritePrecondition, + /// Stream position observed immediately before the append was attempted. + current_position: Option, + }, + /// Zero events were appended to a stream that has no current position, so + /// no [`StreamPosition`] exists to report as the result of the append. + #[error("cannot append zero events to stream '{stream_id}' with no current position")] + EmptyAppendWithoutPosition { + /// Stream identity that the empty append targeted. + stream_id: String, + }, +} + +fn describe_current_position(current_position: &Option) -> String { + match current_position { + Some(position) => format!("current position is {position}"), + None => "stream has no current position".to_string(), + } +} + +fn check_write_precondition( + stream_id: &str, + precondition: StreamWritePrecondition, + current_position: Option, +) -> Result<(), StreamAppendError> { + let satisfied = match precondition { + StreamWritePrecondition::Any => true, + StreamWritePrecondition::StreamExists => current_position.is_some(), + StreamWritePrecondition::NoStream => current_position.is_none(), + StreamWritePrecondition::At(expected) => current_position == Some(expected), + }; + + if satisfied { + Ok(()) + } else { + Err(StreamAppendError::WriteConflict { + stream_id: stream_id.to_string(), + expected: precondition, + current_position, + }) + } +} + +impl StreamRead for InMemoryStore +where + StreamId: AsRef + ToString + Send + Sync + ?Sized, +{ + type Error = std::convert::Infallible; + + async fn read_stream(&self, request: ReadStreamRequest<'_, StreamId>) -> Result { + let stream_id = request.stream_id; + let from_sequence = match request.from { + ReadFrom::Beginning => 1, + ReadFrom::Position(position) => position.as_u64(), + }; + + let events = self + .streams + .lock() + .expect("in-memory event store mutex poisoned") + .get(stream_id.as_ref()) + .cloned() + .unwrap_or_default(); + + let current_position = StreamPosition::try_new(events.len() as u64).ok(); + let recorded_at = Utc::now(); + let mut matched = Vec::new(); + for (index, event) in events.into_iter().enumerate() { + let sequence = index as u64 + 1; + if sequence < from_sequence { + continue; + } + matched.push(StreamEvent { + stream_id: stream_id.to_string(), + event, + stream_position: StreamPosition::try_new(sequence).expect("sequence is always non-zero"), + recorded_at, + }); + } + + Ok(ReadStreamResponse { + current_position, + events: matched, + }) + } +} + +impl StreamAppend for InMemoryStore +where + StreamId: AsRef + ToString + Send + Sync + ?Sized, +{ + type Error = StreamAppendError; + + async fn append_stream( + &self, + request: AppendStreamRequest<'_, StreamId>, + ) -> Result { + let stream_id = request.stream_id.as_ref(); + let mut streams = self.streams.lock().expect("in-memory event store mutex poisoned"); + let stored = streams.entry(stream_id.to_string()).or_default(); + let current_position = StreamPosition::try_new(stored.len() as u64).ok(); + + check_write_precondition(stream_id, request.stream_write_precondition, current_position)?; + + if request.events.is_empty() && current_position.is_none() { + return Err(StreamAppendError::EmptyAppendWithoutPosition { + stream_id: stream_id.to_string(), + }); + } + + stored.extend(request.events); + let stream_position = StreamPosition::try_new(stored.len() as u64) + .expect("stream already had events or new events were appended"); + + Ok(AppendStreamResponse { stream_position }) + } +} + +impl SnapshotRead for InMemoryStore +where + SnapshotId: AsRef + Send + Sync + ?Sized, + Payload: SnapshotPayloadDecode + SnapshotType + Send, + ::Error: std::error::Error + Send + Sync + 'static, + ::Error: std::error::Error + Send + Sync + 'static, +{ + type Error = SnapshotDecodeError<::Error, ::Error>; + + async fn read_snapshot( + &self, + request: ReadSnapshotRequest<'_, SnapshotId>, + ) -> Result, Self::Error> { + let stored = self + .snapshots + .lock() + .expect("in-memory snapshot store mutex poisoned") + .get(request.snapshot_id.as_ref()) + .cloned(); + + let Some(encoded) = stored else { + return Ok(ReadSnapshotResponse { snapshot: None }); + }; + + let snapshot = decode_snapshot::(encoded)?; + Ok(ReadSnapshotResponse { + snapshot: Some(snapshot), + }) + } +} + +impl SnapshotWrite for InMemoryStore +where + SnapshotId: AsRef + Send + Sync + ?Sized, + Payload: SnapshotPayloadEncode + SnapshotType + Send, + ::Error: std::error::Error + Send + Sync + 'static, + ::Error: std::error::Error + Send + Sync + 'static, +{ + type Error = SnapshotEncodeError<::Error, ::Error>; + + async fn write_snapshot( + &self, + request: WriteSnapshotRequest<'_, Payload, SnapshotId>, + ) -> Result { + let encoded = encode_snapshot(&request.snapshot)?; + self.snapshots + .lock() + .expect("in-memory snapshot store mutex poisoned") + .insert(request.snapshot_id.as_ref().to_string(), encoded); + Ok(WriteSnapshotResponse) + } +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-runtime/src/memory/tests.rs b/rsworkspace/crates/trogon-decider-runtime/src/memory/tests.rs new file mode 100644 index 000000000..4c80fb5a4 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-runtime/src/memory/tests.rs @@ -0,0 +1,352 @@ +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::*; +use crate::snapshot::SnapshotPayloadData; +use crate::{EventId, Headers, InvalidSnapshotTypeName, Snapshot, SnapshotTypeName}; + +fn position(value: u64) -> StreamPosition { + StreamPosition::try_new(value).expect("test stream position must be non-zero") +} + +fn event(id: u128, content: &[u8]) -> Event { + Event { + id: EventId::from(Uuid::from_u128(id)), + r#type: "test.event.v1".to_string(), + content: content.to_vec(), + headers: Headers::empty(), + } +} + +fn append( + store: &InMemoryStore, + stream_id: &str, + precondition: StreamWritePrecondition, + events: Vec, +) -> Result { + futures::executor::block_on(store.append_stream(AppendStreamRequest { + stream_id, + stream_write_precondition: precondition, + events, + })) +} + +fn read(store: &InMemoryStore, stream_id: &str, from: ReadFrom) -> ReadStreamResponse { + futures::executor::block_on(store.read_stream(ReadStreamRequest { stream_id, from })).expect("read never fails") +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +struct TestPayload { + label: String, +} + +impl SnapshotPayloadEncode for TestPayload { + type Error = serde_json::Error; + + fn encode(&self) -> Result, Self::Error> { + serde_json::to_vec(self) + } +} + +impl SnapshotPayloadDecode for TestPayload { + type Error = serde_json::Error; + + fn decode(payload: SnapshotPayloadData<'_>) -> Result { + serde_json::from_slice(payload.payload) + } +} + +impl SnapshotType for TestPayload { + type Error = InvalidSnapshotTypeName; + + fn snapshot_type() -> Result { + SnapshotTypeName::new("test.memory.v1.Snapshot") + } +} + +fn write_snapshot( + store: &InMemoryStore, + snapshot_id: &str, + snapshot: Snapshot, +) -> Result> { + futures::executor::block_on(store.write_snapshot(WriteSnapshotRequest { snapshot_id, snapshot })) +} + +fn read_snapshot( + store: &InMemoryStore, + snapshot_id: &str, +) -> Result, SnapshotDecodeError> { + futures::executor::block_on(store.read_snapshot(ReadSnapshotRequest { snapshot_id })) +} + +#[test] +fn read_stream_returns_no_current_position_for_an_unknown_stream() { + let store = InMemoryStore::new(); + + let response = read(&store, "unknown", ReadFrom::Beginning); + + assert_eq!(response.current_position, None); + assert!(response.events.is_empty()); +} + +#[test] +fn append_stream_creates_a_fresh_stream_with_no_stream_precondition() { + let store = InMemoryStore::new(); + + let response = append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one"), event(2, b"two")], + ) + .expect("append to a fresh stream must succeed"); + + assert_eq!(response.stream_position, position(2)); + + let read = read(&store, "orders/1", ReadFrom::Beginning); + assert_eq!(read.current_position, Some(position(2))); + assert_eq!(read.events.len(), 2); + assert_eq!(read.events[0].stream_position, position(1)); + assert_eq!(read.events[1].stream_position, position(2)); + assert_eq!(read.events[0].stream_id, "orders/1"); +} + +#[test] +fn append_stream_rejects_no_stream_precondition_when_stream_already_exists() { + let store = InMemoryStore::new(); + append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one")], + ) + .expect("first append must succeed"); + + let error = append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(2, b"two")], + ) + .expect_err("second append with NoStream must be rejected"); + + assert_eq!( + error, + StreamAppendError::WriteConflict { + stream_id: "orders/1".to_string(), + expected: StreamWritePrecondition::NoStream, + current_position: Some(position(1)), + } + ); +} + +#[test] +fn append_stream_rejects_stream_exists_precondition_when_stream_is_missing() { + let store = InMemoryStore::new(); + + let error = append( + &store, + "orders/1", + StreamWritePrecondition::StreamExists, + vec![event(1, b"one")], + ) + .expect_err("StreamExists must be rejected for a stream with no events"); + + assert_eq!( + error, + StreamAppendError::WriteConflict { + stream_id: "orders/1".to_string(), + expected: StreamWritePrecondition::StreamExists, + current_position: None, + } + ); +} + +#[test] +fn append_stream_accepts_stream_exists_precondition_once_the_stream_has_events() { + let store = InMemoryStore::new(); + append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one")], + ) + .expect("first append must succeed"); + + let response = append( + &store, + "orders/1", + StreamWritePrecondition::StreamExists, + vec![event(2, b"two")], + ) + .expect("StreamExists must succeed once the stream has events"); + + assert_eq!(response.stream_position, position(2)); +} + +#[test] +fn append_stream_rejects_at_precondition_when_position_has_moved() { + let store = InMemoryStore::new(); + append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one")], + ) + .expect("first append must succeed"); + append( + &store, + "orders/1", + StreamWritePrecondition::At(position(1)), + vec![event(2, b"two")], + ) + .expect("second append at the observed position must succeed"); + + let error = append( + &store, + "orders/1", + StreamWritePrecondition::At(position(1)), + vec![event(3, b"three")], + ) + .expect_err("stale At precondition must be rejected"); + + assert_eq!( + error, + StreamAppendError::WriteConflict { + stream_id: "orders/1".to_string(), + expected: StreamWritePrecondition::At(position(1)), + current_position: Some(position(2)), + } + ); +} + +#[test] +fn append_stream_accepts_any_precondition_regardless_of_stream_state() { + let store = InMemoryStore::new(); + + append(&store, "orders/1", StreamWritePrecondition::Any, vec![event(1, b"one")]) + .expect("Any must succeed on a missing stream"); + let response = append(&store, "orders/1", StreamWritePrecondition::Any, vec![event(2, b"two")]) + .expect("Any must succeed on an existing stream"); + + assert_eq!(response.stream_position, position(2)); +} + +#[test] +fn append_stream_rejects_empty_events_against_a_stream_with_no_current_position() { + let store = InMemoryStore::new(); + + let error = append(&store, "orders/1", StreamWritePrecondition::NoStream, Vec::new()) + .expect_err("appending zero events without a current position must be rejected"); + + assert_eq!( + error, + StreamAppendError::EmptyAppendWithoutPosition { + stream_id: "orders/1".to_string(), + } + ); +} + +#[test] +fn append_stream_accepts_empty_events_when_a_current_position_already_exists() { + let store = InMemoryStore::new(); + append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one")], + ) + .expect("first append must succeed"); + + let response = append(&store, "orders/1", StreamWritePrecondition::Any, Vec::new()) + .expect("empty append against an existing position must succeed"); + + assert_eq!(response.stream_position, position(1)); +} + +#[test] +fn read_stream_filters_events_starting_at_the_requested_position() { + let store = InMemoryStore::new(); + append( + &store, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one"), event(2, b"two"), event(3, b"three")], + ) + .expect("append must succeed"); + + let response = read(&store, "orders/1", ReadFrom::Position(position(2))); + + assert_eq!(response.events.len(), 2); + assert_eq!(response.events[0].stream_position, position(2)); + assert_eq!(response.events[1].stream_position, position(3)); +} + +#[test] +fn cloning_the_store_shares_the_same_underlying_state() { + let store = InMemoryStore::new(); + let handle = store.clone(); + + append( + &handle, + "orders/1", + StreamWritePrecondition::NoStream, + vec![event(1, b"one")], + ) + .expect("append through the cloned handle must succeed"); + + let response = read(&store, "orders/1", ReadFrom::Beginning); + assert_eq!(response.current_position, Some(position(1))); +} + +#[test] +fn read_snapshot_returns_none_when_no_snapshot_was_written() { + let store = InMemoryStore::new(); + + let response = read_snapshot(&store, "orders/1").expect("read must succeed"); + + assert!(response.snapshot.is_none()); +} + +#[test] +fn snapshot_round_trips_through_write_and_read() { + let store = InMemoryStore::new(); + let snapshot = Snapshot::new( + position(3), + TestPayload { + label: "checkpoint".to_string(), + }, + ); + + write_snapshot(&store, "orders/1", snapshot.clone()).expect("write must succeed"); + let response = read_snapshot(&store, "orders/1").expect("read must succeed"); + + assert_eq!(response.snapshot, Some(snapshot)); +} + +#[test] +fn snapshot_write_overwrites_the_previous_snapshot_for_the_same_id() { + let store = InMemoryStore::new(); + write_snapshot( + &store, + "orders/1", + Snapshot::new( + position(1), + TestPayload { + label: "first".to_string(), + }, + ), + ) + .expect("first write must succeed"); + + let second = Snapshot::new( + position(2), + TestPayload { + label: "second".to_string(), + }, + ); + write_snapshot(&store, "orders/1", second.clone()).expect("second write must succeed"); + + let response = read_snapshot(&store, "orders/1").expect("read must succeed"); + assert_eq!(response.snapshot, Some(second)); +} diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/encoded_snapshot.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/encoded_snapshot.rs index a6caf2d9e..d68ecb83d 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/encoded_snapshot.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/encoded_snapshot.rs @@ -7,14 +7,23 @@ use super::{ SnapshotPayloadData, SnapshotPayloadDecode, SnapshotPayloadEncode, }; +/// A snapshot payload after it has been encoded to bytes but before it is +/// serialized to the wire envelope, or after the envelope has been parsed but +/// before the payload bytes are decoded back into a decider state. #[derive(Debug, Clone, PartialEq, Eq)] pub struct EncodedSnapshot { + /// Stable snapshot type name, used to reject a payload encoded for a + /// different decider state. pub r#type: String, + /// The stream position covered by the snapshot payload. pub position: StreamPosition, + /// The encoded decider state bytes. pub payload: Vec, } impl EncodedSnapshot { + /// Assembles an encoded snapshot from its type name, position, and + /// already-encoded payload bytes. pub fn new(snapshot_type: impl Into, position: StreamPosition, payload: Vec) -> Self { Self { r#type: snapshot_type.into(), @@ -23,6 +32,7 @@ impl EncodedSnapshot { } } + /// Serializes this snapshot into its on-the-wire envelope bytes. pub fn into_bytes(self) -> Result, SnapshotEnvelopeEncodeError> { serde_json::to_vec(&SnapshotEnvelope { r#type: self.r#type, @@ -32,6 +42,7 @@ impl EncodedSnapshot { .map_err(SnapshotEnvelopeEncodeError::new) } + /// Parses an on-the-wire envelope back into an encoded snapshot. pub fn from_bytes(value: &[u8]) -> Result { let envelope = serde_json::from_slice::(value).map_err(SnapshotEnvelopeDecodeError::envelope_source)?; @@ -42,6 +53,8 @@ impl EncodedSnapshot { } } +/// Encodes a decider state snapshot's payload and resolves its type name, +/// producing a storage-neutral [`EncodedSnapshot`]. pub fn encode_snapshot( snapshot: &Snapshot, ) -> Result::Error, ::Error>> @@ -54,6 +67,8 @@ where Ok(EncodedSnapshot::new(snapshot_type.as_str(), snapshot.position, payload)) } +/// Decodes an [`EncodedSnapshot`] back into a typed decider state snapshot, +/// rejecting the payload if its stored type name does not match `T`'s. pub fn decode_snapshot( encoded: EncodedSnapshot, ) -> Result, SnapshotDecodeError<::Error, ::Error>> diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_decode_error.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_decode_error.rs index 67fdb4ef5..6b3d260c7 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_decode_error.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_decode_error.rs @@ -1,19 +1,32 @@ use crate::SnapshotTypeName; +/// Error returned by [`decode_snapshot`](super::decode_snapshot) when a +/// snapshot cannot be turned back into a typed decider state. #[derive(Debug, thiserror::Error)] pub enum SnapshotDecodeError { + /// The decider state's own snapshot type name could not be resolved. #[error("failed to resolve snapshot type: {source}")] SnapshotType { + /// The underlying [`SnapshotType`](crate::SnapshotType) failure. #[source] source: SnapshotTypeSource, }, + /// The encoded payload bytes could not be decoded into the decider state. #[error("failed to decode snapshot payload: {source}")] Payload { + /// The underlying [`SnapshotPayloadDecode`](super::SnapshotPayloadDecode) failure. #[source] source: PayloadSource, }, + /// The snapshot's stored type name does not match the decider state's + /// expected type name, so decoding was refused before touching the payload. #[error("unexpected snapshot type: expected {expected}, got {actual}")] - UnexpectedType { expected: SnapshotTypeName, actual: String }, + UnexpectedType { + /// Type name the decider state expected. + expected: SnapshotTypeName, + /// Type name actually stored on the snapshot. + actual: String, + }, } impl SnapshotDecodeError { @@ -29,6 +42,7 @@ impl SnapshotDecodeError Option<&PayloadSource> { match self { Self::Payload { source } => Some(source), @@ -36,6 +50,7 @@ impl SnapshotDecodeError Option<&SnapshotTypeSource> { match self { Self::SnapshotType { source } => Some(source), diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_encode_error.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_encode_error.rs index c0c8c3ad5..ccbc5a4a4 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_encode_error.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_encode_error.rs @@ -1,12 +1,18 @@ +/// Error returned by [`encode_snapshot`](super::encode_snapshot) when a +/// decider state cannot be turned into an [`EncodedSnapshot`](super::EncodedSnapshot). #[derive(Debug, thiserror::Error)] pub enum SnapshotEncodeError { + /// The decider state's own snapshot type name could not be resolved. #[error("failed to resolve snapshot type: {source}")] SnapshotType { + /// The underlying [`SnapshotType`](crate::SnapshotType) failure. #[source] source: SnapshotTypeSource, }, + /// The decider state could not be encoded into payload bytes. #[error("failed to encode snapshot payload: {source}")] Payload { + /// The underlying [`SnapshotPayloadEncode`](super::SnapshotPayloadEncode) failure. #[source] source: PayloadSource, }, @@ -21,6 +27,7 @@ impl SnapshotEncodeError Option<&PayloadSource> { match self { Self::Payload { source } => Some(source), @@ -28,6 +35,7 @@ impl SnapshotEncodeError Option<&SnapshotTypeSource> { match self { Self::SnapshotType { source } => Some(source), diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_decode_error.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_decode_error.rs index 5cdb3bab1..a10ea6a09 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_decode_error.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_decode_error.rs @@ -1,14 +1,20 @@ use crate::InvalidStreamPosition; +/// Error returned by [`EncodedSnapshot::from_bytes`](super::EncodedSnapshot::from_bytes) +/// when the on-the-wire envelope cannot be parsed back into an [`EncodedSnapshot`](super::EncodedSnapshot). #[derive(Debug, thiserror::Error)] pub enum SnapshotEnvelopeDecodeError { + /// The envelope bytes are not valid JSON for the expected shape. #[error("failed to decode snapshot envelope: {source}")] Envelope { + /// The underlying JSON deserialization failure. #[source] source: serde_json::Error, }, + /// The envelope's stored position is not a valid [`StreamPosition`](crate::StreamPosition). #[error("failed to decode snapshot position: {source}")] Position { + /// The underlying position validation failure. #[source] source: InvalidStreamPosition, }, diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_encode_error.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_encode_error.rs index 4b45ff779..81b3db21c 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_encode_error.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_envelope_encode_error.rs @@ -1,3 +1,6 @@ +/// Error returned by [`EncodedSnapshot::into_bytes`](super::EncodedSnapshot::into_bytes) +/// when an [`EncodedSnapshot`](super::EncodedSnapshot) cannot be serialized to its +/// on-the-wire envelope bytes. #[derive(Debug, thiserror::Error)] #[error("failed to encode snapshot envelope: {source}")] pub struct SnapshotEnvelopeEncodeError { diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_decode.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_decode.rs index 07acd9170..0a6bc4edf 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_decode.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_decode.rs @@ -1,17 +1,26 @@ +/// Encoded snapshot payload bytes handed to [`SnapshotPayloadDecode::decode`]. +/// +/// `#[non_exhaustive]` so new fields can be added later without breaking +/// callers that construct or match on this type. #[derive(Debug, Clone, Copy)] #[non_exhaustive] pub struct SnapshotPayloadData<'a> { + /// The encoded payload bytes to decode. pub payload: &'a [u8], } impl<'a> SnapshotPayloadData<'a> { + /// Wraps encoded payload bytes for a decode call. pub const fn new(payload: &'a [u8]) -> Self { Self { payload } } } +/// Decodes a decider state from its encoded snapshot payload bytes. pub trait SnapshotPayloadDecode: Sized { + /// Error returned when the payload bytes cannot be decoded. type Error: std::error::Error + Send + Sync + 'static; + /// Decodes a decider state from the given payload bytes. fn decode(payload: SnapshotPayloadData<'_>) -> Result; } diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_encode.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_encode.rs index cd4dc6a9f..9fb787ff3 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_encode.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/codec/snapshot_payload_encode.rs @@ -1,5 +1,8 @@ +/// Encodes a decider state into bytes suitable for a snapshot payload. pub trait SnapshotPayloadEncode { + /// Error returned when the state cannot be encoded. type Error: std::error::Error + Send + Sync + 'static; + /// Encodes this decider state into payload bytes. fn encode(&self) -> Result, Self::Error>; } diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/mod.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/mod.rs index 87885d6ee..c59b98b53 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/mod.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/mod.rs @@ -14,6 +14,8 @@ pub use read_snapshot::{ReadSnapshotRequest, ReadSnapshotResponse, SnapshotRead} pub use snapshot_type::{InvalidSnapshotTypeName, SnapshotType, SnapshotTypeName}; pub use write_snapshot::{SnapshotWrite, WriteSnapshotRequest, WriteSnapshotResponse}; +/// A point-in-time capture of decider state, tagged with the stream position +/// it was taken at so replay can resume strictly after it. #[derive(Debug, Clone, PartialEq)] pub struct Snapshot { /// The stream position covered by the snapshot payload. @@ -22,10 +24,12 @@ pub struct Snapshot { /// `StreamPosition`, not a revision, because adapters are allowed to use /// sparse but comparable positions. pub position: StreamPosition, + /// The captured decider state. pub payload: T, } impl Snapshot { + /// Pairs a decider state with the stream position it was captured at. pub fn new(position: StreamPosition, payload: T) -> Self { Self { position, payload } } diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/read_snapshot.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/read_snapshot.rs index e0fac5ac0..c31f64644 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/read_snapshot.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/read_snapshot.rs @@ -1,24 +1,33 @@ use super::Snapshot; +/// Request to load the most recent snapshot for one snapshot identity. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ReadSnapshotRequest<'a, SnapshotId: ?Sized> { + /// Snapshot identity in the caller's domain-specific representation. pub snapshot_id: &'a SnapshotId, } +/// Result of loading a snapshot. #[derive(Debug, Clone, PartialEq)] pub struct ReadSnapshotResponse { + /// The stored snapshot, or `None` if no snapshot exists yet for the + /// requested identity. pub snapshot: Option>, } impl ReadSnapshotResponse { + /// Extracts the loaded snapshot, discarding the response wrapper. pub fn into_snapshot(self) -> Option> { self.snapshot } } +/// Loads snapshots for a decider state from a backing store. pub trait SnapshotRead: Send + Sync { + /// Backend-specific read error. type Error: std::error::Error + Send + Sync + 'static; + /// Reads the most recent snapshot for the requested identity. fn read_snapshot( &self, request: ReadSnapshotRequest<'_, SnapshotId>, diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/snapshot_type.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/snapshot_type.rs index af3c35fc6..f4006f210 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/snapshot_type.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/snapshot_type.rs @@ -97,9 +97,14 @@ pub enum InvalidSnapshotTypeName { ContainsControlCharacter, } +/// Associates a decider state type with the stable [`SnapshotTypeName`] its +/// snapshots are tagged with, so a decoder can reject a snapshot payload +/// encoded for a different state type. pub trait SnapshotType { + /// Error returned when the type name cannot be resolved or constructed. type Error: std::error::Error + Send + Sync + 'static; + /// Returns the stable type name for this state's snapshots. fn snapshot_type() -> Result; } diff --git a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/write_snapshot.rs b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/write_snapshot.rs index b7ddf6947..ca1566301 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/snapshot/write_snapshot.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/snapshot/write_snapshot.rs @@ -1,17 +1,25 @@ use super::Snapshot; +/// Request to persist a snapshot for one snapshot identity. #[derive(Debug, Clone, PartialEq)] pub struct WriteSnapshotRequest<'a, SnapshotPayload, SnapshotId: ?Sized> { + /// Snapshot identity in the caller's domain-specific representation. pub snapshot_id: &'a SnapshotId, + /// The snapshot to persist. pub snapshot: Snapshot, } +/// Result of a successful snapshot write. Carries no data; its existence +/// distinguishes success from the store's own error type. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct WriteSnapshotResponse; +/// Persists snapshots for a decider state to a backing store. pub trait SnapshotWrite: Send + Sync { + /// Backend-specific write error. type Error: std::error::Error + Send + Sync + 'static; + /// Persists the given snapshot for the requested identity. fn write_snapshot( &self, request: WriteSnapshotRequest<'_, SnapshotPayload, SnapshotId>, diff --git a/rsworkspace/crates/trogon-decider-runtime/src/stream/read_stream.rs b/rsworkspace/crates/trogon-decider-runtime/src/stream/read_stream.rs index e303aa676..471cc2974 100644 --- a/rsworkspace/crates/trogon-decider-runtime/src/stream/read_stream.rs +++ b/rsworkspace/crates/trogon-decider-runtime/src/stream/read_stream.rs @@ -9,7 +9,9 @@ use crate::StreamEvent; /// "start at" primitive. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ReadFrom { + /// Reads from the first event in the stream. Beginning, + /// Reads from the given position, inclusive. Position(StreamPosition), } diff --git a/rsworkspace/crates/trogon-decider-sim/Cargo.toml b/rsworkspace/crates/trogon-decider-sim/Cargo.toml index 4c0d5662d..f4d474659 100644 --- a/rsworkspace/crates/trogon-decider-sim/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-sim/Cargo.toml @@ -5,15 +5,28 @@ edition = "2024" license = "Apache-2.0" publish = false -[lints] -workspace = true +[lints.rust] +missing_docs = "warn" +warnings = "deny" +unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(dylint_lib, values(any()))'] } + +[lints.clippy] +all = "deny" +expect_used = "deny" +panic = "deny" +unwrap_used = "deny" [features] -test-support = [] +test-support = ["dep:buffa", "dep:trogon-decider", "dep:trogon-decider-guest-sdk"] [dependencies] +buffa = { workspace = true, optional = true } tempfile = "3" thiserror = { workspace = true } +tracing = { workspace = true } +trogon-decider = { workspace = true, optional = true } +trogon-decider-guest-sdk = { workspace = true, optional = true } +trogon-decider-wasm-runtime = { workspace = true } trogon-decider-wit = { workspace = true, features = ["host"] } trogon-std = { workspace = true } trogonai-proto = { path = "../trogonai-proto", default-features = false, features = ["schedules"] } @@ -23,7 +36,9 @@ wasmtime = { version = "=46.0.1", default-features = false, features = ["anyhow" [dev-dependencies] buffa = { workspace = true } buffa-types = { workspace = true } +trogon-decider-test = { path = "../../cli/trogon-decider-test" } trogon-decider-wit = { workspace = true, features = ["host"] } +trogon-scheduler-domain = { path = "../trogon-scheduler-domain" } wasmtime = { version = "=46.0.1", default-features = false, features = ["anyhow", "component-model", "runtime", "cranelift"] } wat = "1" @@ -31,3 +46,8 @@ wat = "1" name = "schedules" path = "tests/schedules.rs" required-features = ["test-support"] + +[[test]] +name = "parity" +path = "tests/parity.rs" +required-features = ["test-support"] diff --git a/rsworkspace/crates/trogon-decider-sim/README.md b/rsworkspace/crates/trogon-decider-sim/README.md new file mode 100644 index 000000000..e0be43546 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/README.md @@ -0,0 +1,22 @@ +# trogon-decider-sim + +Per [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md), this crate is a testing-only +adapter around the same wasmtime host machinery `trogon-decider-wasm-runtime` uses in +production, so a decider's guest behavior can be exercised in-process without a real +deployment. `SimHost`/`SimInstance` load a component and run every guest call against the same +production resource budget (`WasmEngineConfig`) `trogon-decider-wasm-runtime` enforces, so a +guest that would trap or exceed its deadline in production does the same in a test rather than +running unbounded. `SimScenario` is a fluent given/when/then builder for driving one guest +session through an ordered sequence of commands. `ScenarioIr`/`WireEnvelope` (`ir.rs`) express a +decider-agnostic scenario in wire form so the same scenario can run against both a wasm +component and, with the `test-support` feature, a native `Decider` implementation +(`native.rs`), letting `assert_parity` (`parity.rs`) fail a test on the first divergence +between the two. + +`assert_zero_imports` (`import_check.rs`) checks a compiled component declares no imports via +`wasm-tools`, matching the same requirement `WasmDeciderModule::load` enforces structurally at +load time. + +See `tests/schedules.rs` for a worked example, and +[Decider Platform](../../../docs/architecture/decider.md) for how this harness relates to +`TestCase` and the `decider-test` CLI. diff --git a/rsworkspace/crates/trogon-decider-sim/src/fixture.rs b/rsworkspace/crates/trogon-decider-sim/src/fixture.rs index 122f47d83..5727c57ff 100644 --- a/rsworkspace/crates/trogon-decider-sim/src/fixture.rs +++ b/rsworkspace/crates/trogon-decider-sim/src/fixture.rs @@ -9,6 +9,8 @@ pub struct SimFixture { } impl SimFixture { + /// Loads the pre-built `trogon_schedules_decider.wasm` release artifact from the workspace + /// target directory. pub fn schedules() -> Self { Self::load( "trogon_schedules_decider.wasm", @@ -16,6 +18,7 @@ impl SimFixture { ) } + /// Returns the fixture's compiled component bytes. pub fn bytes(&self) -> &[u8] { &self.bytes } diff --git a/rsworkspace/crates/trogon-decider-sim/src/host.rs b/rsworkspace/crates/trogon-decider-sim/src/host.rs index 8a3aeacb8..fde9fb7ee 100644 --- a/rsworkspace/crates/trogon-decider-sim/src/host.rs +++ b/rsworkspace/crates/trogon-decider-sim/src/host.rs @@ -1,83 +1,170 @@ use thiserror::Error; +use trogon_decider_wasm_runtime::{WasmDeciderEngine, WasmEngineConfig, WasmEngineError}; use trogon_decider_wit::host::{self, Decider}; use wasmtime::component::{Component, Linker}; -use wasmtime::{Config, Engine, Store}; +use wasmtime::{Store, StoreLimits, StoreLimitsBuilder}; use crate::import_check::{ImportCheckError, assert_zero_imports}; +/// Failure loading, instantiating, or arming a [`SimHost`]'s wasm component. #[derive(Debug, Error)] pub enum SimError { + /// The compiled component declares an import, so it cannot be treated as a production-shaped + /// zero-import decider. #[error(transparent)] Import(#[from] ImportCheckError), + /// The wasmtime engine could not be configured with the requested resource budget. #[error("failed to configure wasmtime engine")] - Engine { - #[source] - source: wasmtime::Error, - }, + Engine(#[from] WasmEngineError), + /// The component bytes failed to compile. #[error("failed to compile wasm component")] Compile { + /// The underlying wasmtime compilation error. #[source] source: wasmtime::Error, }, + /// The compiled component failed to instantiate against the linker. #[error("failed to instantiate wasm component")] Instantiate { + /// The underlying wasmtime instantiation error. + #[source] + source: wasmtime::Error, + }, + /// The store's fuel or epoch deadline could not be armed before a guest call. + #[error("failed to arm guest call resource budget")] + Arm { + /// The underlying wasmtime error setting fuel or epoch deadline. #[source] source: wasmtime::Error, }, } /// Loaded decider component ready to instantiate in-memory test sessions. +/// +/// Every guest call runs against the same resource budget +/// [`trogon_decider_wasm_runtime::WasmDeciderEngine`] applies in production: +/// fuel metering, epoch-based wall-clock interruption, and pooled +/// memory/table/instance ceilings. A decider that would trap in production +/// traps the same way here, instead of running unbounded in sim and CI. pub struct SimHost { - engine: Engine, + engine: WasmDeciderEngine, component: Component, } impl SimHost { + /// Loads a decider component with the default production resource budget. pub fn load(bytes: &[u8]) -> Result { + Self::load_with_config(bytes, WasmEngineConfig::default()) + } + + /// Loads a decider component with an explicit resource budget, letting + /// tests exercise trapping behavior (fuel exhaustion, epoch interruption, + /// memory ceilings) deterministically. + pub fn load_with_config(bytes: &[u8], config: WasmEngineConfig) -> Result { assert_zero_imports(bytes)?; - let mut config = Config::new(); - config.wasm_component_model(true); - let engine = Engine::new(&config).map_err(|source| SimError::Engine { source })?; - let component = Component::new(&engine, bytes).map_err(|source| SimError::Compile { source })?; + let engine = WasmDeciderEngine::new(config)?; + let component = Component::new(engine.engine(), bytes).map_err(|source| SimError::Compile { source })?; Ok(Self { engine, component }) } + /// Returns the resource budget applied to every guest store created from + /// this host. + pub fn config(&self) -> WasmEngineConfig { + self.engine.config() + } + + /// Creates a new guest store for `host_state`, arms its resource budget, and instantiates the + /// component's `Decider` bindings into it. pub fn instantiate(&self, host_state: T) -> Result, SimError> { - let linker = Linker::new(&self.engine); - let mut store = Store::new(&self.engine, host_state); + let config = self.engine.config(); + let linker = Linker::new(self.engine.engine()); + let mut store = Store::new(self.engine.engine(), SimGuestState::new(&config, host_state)); + store.limiter(|state| &mut state.limits); + arm_guest_call(&mut store, &config)?; let bindings = Decider::instantiate(&mut store, &self.component, &linker) .map_err(|source| SimError::Instantiate { source })?; - Ok(SimInstance { store, bindings }) + Ok(SimInstance { + store, + bindings, + config, + }) } } +/// Per-session store state: the resource limiter production installs plus the +/// caller-supplied host state. +pub(crate) struct SimGuestState { + pub(crate) limits: StoreLimits, + #[allow( + dead_code, + reason = "kept as the store's data so future host functions can read it via store.data()" + )] + pub(crate) host_state: T, +} + +impl SimGuestState { + fn new(config: &WasmEngineConfig, host_state: T) -> Self { + Self { + limits: StoreLimitsBuilder::new() + .memory_size(config.max_memory_bytes()) + .table_elements(config.max_table_elements()) + .instances(config.max_instances_per_session()) + .tables(config.max_tables_per_session()) + .memories(config.max_memories_per_session()) + .build(), + host_state, + } + } +} + +/// Arms a guest export call with the same fuel budget and wall-clock epoch +/// deadline production sets immediately before every guest export call. +pub(crate) fn arm_guest_call(store: &mut Store, config: &WasmEngineConfig) -> Result<(), SimError> { + store + .set_fuel(config.fuel_per_call()) + .map_err(|source| SimError::Arm { source })?; + store.set_epoch_deadline(config.epoch_ticks_per_call()); + Ok(()) +} + +/// A single instantiated guest component paired with its store and resource budget, ready to +/// open sessions and issue guest calls against. pub struct SimInstance { - pub(crate) store: Store, + pub(crate) store: Store>, pub(crate) bindings: Decider, + pub(crate) config: WasmEngineConfig, } impl SimInstance { + /// Arms the guest call budget and returns the component's declared module descriptor. pub fn descriptor(&mut self) -> Result { + arm_guest_call(&mut self.store, &self.config)?; host::call_descriptor(&self.bindings, &mut self.store) } + /// Arms the guest call budget and resolves the stream id `command` targets. pub fn stream_id( &mut self, command: &host::CommandEnvelope, ) -> Result, wasmtime::Error> { + arm_guest_call(&mut self.store, &self.config)?; host::call_stream_id(&self.bindings, &mut self.store, command) } + /// Arms the guest call budget and opens a new guest session, optionally seeded from a prior + /// `snapshot`. pub fn open_session( &mut self, snapshot: Option<&[u8]>, ) -> Result, wasmtime::Error> { + arm_guest_call(&mut self.store, &self.config)?; let session = host::create_session(&self.bindings, &mut self.store, snapshot)?; Ok(super::session::SimSession { bindings: &self.bindings, store: &mut self.store, + config: self.config, session, }) } diff --git a/rsworkspace/crates/trogon-decider-sim/src/import_check.rs b/rsworkspace/crates/trogon-decider-sim/src/import_check.rs index 30516eeaf..a863a1713 100644 --- a/rsworkspace/crates/trogon-decider-sim/src/import_check.rs +++ b/rsworkspace/crates/trogon-decider-sim/src/import_check.rs @@ -4,14 +4,27 @@ use std::process::Command; use thiserror::Error; use trogon_std::env::{ReadEnv as _, SystemEnv}; +/// Failure verifying that a compiled component declares no imports. #[derive(Debug, Error)] pub enum ImportCheckError { + /// The component's core wasm import section names an import (wasmparser fallback path). #[error("component declares import '{module}/{name}'")] - ImportPresent { module: String, name: String }, + ImportPresent { + /// The imported module name. + module: String, + /// The imported item name within `module`. + name: String, + }, + /// The component's decompiled WIT declares an `import` line (wasm-tools path). #[error("component WIT declares import: {line}")] - WorldImport { line: String }, + WorldImport { + /// The offending `import` line, trimmed of leading whitespace. + line: String, + }, + /// The wasm bytes could not be parsed as a module or component. #[error("failed to parse wasm module: {0}")] Parse(String), + /// The `wasm-tools` binary could not be located or invoked. #[error("wasm-tools not available: {0}")] WasmToolsUnavailable(String), } diff --git a/rsworkspace/crates/trogon-decider-sim/src/ir.rs b/rsworkspace/crates/trogon-decider-sim/src/ir.rs new file mode 100644 index 000000000..dd764e4cf --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/src/ir.rs @@ -0,0 +1,245 @@ +//! Decider-agnostic scenario representation shared by every runner. +//! +//! A [`ScenarioIr`] captures a given/when/then scenario purely in wire form: type URLs and +//! encoded payload bytes, with no dependency on any specific decider's Rust types. Both the +//! wasm runner ([`ScenarioIr::to_sim_scenario`], driving a compiled component through +//! [`SimScenario`], and [`ScenarioIr::run_wasm`], capturing a component's raw outcomes) and a +//! native runner (see the `native` module, gated behind the `test-support` feature) consume the +//! same `ScenarioIr` value, so one scenario can be executed through either path without being +//! re-specified per path. + +use trogon_decider_wit::host; + +use crate::host::SimInstance; +use crate::scenario::{ScenarioError, SimScenario}; + +/// A wire-form event or command envelope: a type URL plus its encoded payload. +/// +/// Commands are tagged with the full `type.googleapis.com/`-prefixed URL; events are tagged +/// with the bare protobuf message full name. Both runners interpret a `WireEnvelope`'s +/// `type_url` the same way real wire traffic would, so a codec divergence between the wasm +/// guest and a native decider shows up as a type or payload mismatch instead of being silently +/// normalized away. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WireEnvelope { + /// The envelope's type URL: full for commands, bare protobuf full name for events. + pub type_url: String, + /// The encoded payload bytes. + pub payload: Vec, +} + +impl WireEnvelope { + /// Creates a wire envelope from a type URL and payload bytes. + pub fn new(type_url: impl Into, payload: impl Into>) -> Self { + Self { + type_url: type_url.into(), + payload: payload.into(), + } + } +} + +impl From<&WireEnvelope> for host::AnyEnvelope { + fn from(value: &WireEnvelope) -> Self { + Self { + type_: value.type_url.clone(), + payload: value.payload.clone(), + } + } +} + +impl From<&WireEnvelope> for host::CommandEnvelope { + fn from(value: &WireEnvelope) -> Self { + Self { + type_: value.type_url.clone(), + payload: value.payload.clone(), + } + } +} + +impl From for WireEnvelope { + fn from(value: host::AnyEnvelope) -> Self { + Self { + type_url: value.type_, + payload: value.payload, + } + } +} + +impl From for WireEnvelope { + fn from(value: host::CommandEnvelope) -> Self { + Self { + type_url: value.type_, + payload: value.payload, + } + } +} + +/// The declared outcome a [`ScenarioStep`] expects from its `when` command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExpectedOutcome { + /// The command must be accepted and produce exactly these events, in order. + Events(Vec), + /// The command must be rejected by a business rule (not a fault). + Rejected, + /// The command must be accepted, without asserting which events it produces. + Accepted, + /// The command must fail (rejected or faulted) with this code or message. + Error(String), +} + +/// One `when`/`then` pair in a scenario's ordered step sequence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScenarioStep { + /// The command envelope decided against the state accumulated so far. + pub when: WireEnvelope, + /// The outcome this step's `when` command is expected to produce. + pub expect: ExpectedOutcome, +} + +/// The actual outcome one step produced, captured raw rather than checked against a declared +/// [`ExpectedOutcome`]. +/// +/// The parity harness compares a native runner's and a wasm runner's `StepOutcome` values +/// directly against each other, independent of what the scenario declares. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StepOutcome { + /// The command was accepted, producing these events in order. + Events(Vec), + /// The command was rejected by a business rule. + Rejected { + /// The domain error's stable, machine-readable code. + code: String, + /// The domain error's human-readable message. + message: String, + }, + /// The command failed for a reason other than a business rule: an unknown command type, a + /// decode failure, an encode failure, and so on. + Faulted { + /// The domain error's stable, machine-readable code. + code: String, + /// The domain error's human-readable message. + message: String, + }, +} + +/// A decider-agnostic given/when/then scenario, expressed entirely in wire form. +/// +/// `ScenarioIr` is the shared source of truth both runners execute: the CLI parses its YAML +/// `Scenario`/`Step` shapes into this IR, the wasm runner builds a [`SimScenario`] from it via +/// [`ScenarioIr::to_sim_scenario`] (or captures raw outcomes via [`ScenarioIr::run_wasm`]), and a +/// native runner (see the `native` module) executes it directly against a +/// [`trogon_decider::Decider`](https://docs.rs/trogon-decider) implementation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScenarioIr { + /// The scenario's human-readable name, used in test and CLI output. + pub name: String, + /// The stream id every command in this scenario targets, when known ahead of time. + /// + /// Both runners can derive a stream id from a command independently (via `Decider::stream_id` + /// natively, or the WIT `stream-id` export in wasm); when this is `Some`, the parity harness + /// also asserts that derived id matches it. + pub stream_id: Option, + /// The session's seeded history, replayed via `evolve` before the first step's command is + /// decided. + pub given: Vec, + /// The scenario's ordered `when`/`then` steps, run against a single open session. + pub steps: Vec, +} + +impl ScenarioIr { + /// Creates an empty scenario with the given name. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + stream_id: None, + given: Vec::new(), + steps: Vec::new(), + } + } + + /// Returns the command envelope of this scenario's first step, if any. + pub fn first_command(&self) -> Option<&WireEnvelope> { + self.steps.first().map(|step| &step.when) + } + + /// Builds a [`SimScenario`] that asserts each step's declared [`ExpectedOutcome`], ready to + /// run against a wasm component instance via [`SimScenario::run`]. + pub fn to_sim_scenario(&self) -> SimScenario { + let mut sim = SimScenario::new().given(self.given.iter().map(host::AnyEnvelope::from)); + for step in &self.steps { + sim = sim.when(host::CommandEnvelope::from(&step.when)); + sim = match &step.expect { + ExpectedOutcome::Events(events) => sim.then_events(events.iter().map(host::AnyEnvelope::from)), + ExpectedOutcome::Rejected => sim.then_rejected(), + ExpectedOutcome::Accepted => sim.then_accepted(), + ExpectedOutcome::Error(expected) => sim.then_error(expected.clone()), + }; + } + sim + } + + /// Runs this scenario against a wasm component instance, capturing each step's raw + /// [`StepOutcome`] instead of asserting it against the step's declared [`ExpectedOutcome`]. + /// + /// Used by the parity harness to compare the wasm runner's actual behavior against a native + /// runner's. + pub fn run_wasm(&self, instance: &mut SimInstance) -> Result, ScenarioError> { + let mut session = instance + .open_session(None) + .map_err(|source| ScenarioError::OpenSession { source })?; + + if !self.given.is_empty() { + let given: Vec = self.given.iter().map(host::AnyEnvelope::from).collect(); + session + .evolve(&given) + .map_err(|source| ScenarioError::EvolveCall { source })? + .map_err(|err| ScenarioError::Evolve { + code: err.code, + message: err.message, + })?; + } + + let mut outcomes = Vec::with_capacity(self.steps.len()); + let mut forwarded: Vec = Vec::new(); + + for step in &self.steps { + if !forwarded.is_empty() { + session + .evolve(&forwarded) + .map_err(|source| ScenarioError::EvolveCall { source })? + .map_err(|err| ScenarioError::Evolve { + code: err.code, + message: err.message, + })?; + forwarded.clear(); + } + + let command = host::CommandEnvelope::from(&step.when); + let outcome = session + .decide(&command) + .map_err(|source| ScenarioError::DecideCall { source })?; + + let step_outcome = match outcome { + Ok(events) => { + let events: Vec = events.into_iter().map(WireEnvelope::from).collect(); + forwarded = events.iter().map(host::AnyEnvelope::from).collect(); + StepOutcome::Events(events) + } + Err(host::DecideError::Rejected(err)) => StepOutcome::Rejected { + code: err.code, + message: err.message, + }, + Err(host::DecideError::Faulted(err)) => StepOutcome::Faulted { + code: err.code, + message: err.message, + }, + }; + outcomes.push(step_outcome); + } + + Ok(outcomes) + } +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-sim/src/ir/tests.rs b/rsworkspace/crates/trogon-decider-sim/src/ir/tests.rs new file mode 100644 index 000000000..839262fcf --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/src/ir/tests.rs @@ -0,0 +1,62 @@ +use trogon_decider_wit::host; + +use super::*; + +#[test] +fn wire_envelope_round_trips_through_host_any_envelope() { + let wire = WireEnvelope::new("trogonai.scheduler.schedules.v1.ScheduleCreated", vec![1, 2, 3]); + let any = host::AnyEnvelope::from(&wire); + assert_eq!(any.type_, wire.type_url); + assert_eq!(any.payload, wire.payload); + assert_eq!(WireEnvelope::from(any), wire); +} + +#[test] +fn wire_envelope_round_trips_through_host_command_envelope() { + let wire = WireEnvelope::new( + "type.googleapis.com/trogonai.scheduler.schedules.v1.CreateSchedule", + vec![4, 5, 6], + ); + let command = host::CommandEnvelope::from(&wire); + assert_eq!(command.type_, wire.type_url); + assert_eq!(command.payload, wire.payload); + assert_eq!(WireEnvelope::from(command), wire); +} + +#[test] +fn new_scenario_has_no_steps_or_stream_id() { + let scenario = ScenarioIr::new("empty scenario"); + assert_eq!(scenario.name, "empty scenario"); + assert!(scenario.stream_id.is_none()); + assert!(scenario.given.is_empty()); + assert!(scenario.steps.is_empty()); + assert!(scenario.first_command().is_none()); +} + +#[test] +fn first_command_returns_the_first_steps_when() { + let mut scenario = ScenarioIr::new("scenario with steps"); + let first = WireEnvelope::new("a", Vec::new()); + let second = WireEnvelope::new("b", Vec::new()); + scenario.steps.push(ScenarioStep { + when: first.clone(), + expect: ExpectedOutcome::Accepted, + }); + scenario.steps.push(ScenarioStep { + when: second, + expect: ExpectedOutcome::Rejected, + }); + + assert_eq!(scenario.first_command(), Some(&first)); +} + +#[test] +fn to_sim_scenario_builds_without_running() { + let mut scenario = ScenarioIr::new("buildable scenario"); + scenario.steps.push(ScenarioStep { + when: WireEnvelope::new("a", Vec::new()), + expect: ExpectedOutcome::Events(vec![WireEnvelope::new("b", Vec::new())]), + }); + + let _sim = scenario.to_sim_scenario(); +} diff --git a/rsworkspace/crates/trogon-decider-sim/src/lib.rs b/rsworkspace/crates/trogon-decider-sim/src/lib.rs index 203f3ee8e..369934bdc 100644 --- a/rsworkspace/crates/trogon-decider-sim/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-sim/src/lib.rs @@ -3,16 +3,32 @@ mod host; mod import_check; +pub mod ir; mod scenario; mod session; #[cfg(feature = "test-support")] pub mod fixture; +#[cfg(feature = "test-support")] +pub mod native; + +#[cfg(feature = "test-support")] +pub mod parity; + #[cfg(feature = "test-support")] pub use fixture::SimFixture; pub use host::{SimError, SimHost, SimInstance}; pub use import_check::{ImportCheckError, assert_zero_imports}; +pub use ir::{ExpectedOutcome, ScenarioIr, ScenarioStep, StepOutcome, WireEnvelope}; +#[cfg(feature = "test-support")] +pub use native::{ + NativeDecideError, NativeDeciderBundle, NativeDomainError, NativeRunError, decode_native_command, native_decide, + native_evolve_one, +}; +#[cfg(feature = "test-support")] +pub use parity::{ParityError, assert_parity}; pub use scenario::{ScenarioError, SimScenario}; pub use session::SimSession; +pub use trogon_decider_wasm_runtime::{WasmEngineConfig, WasmEngineError}; diff --git a/rsworkspace/crates/trogon-decider-sim/src/native.rs b/rsworkspace/crates/trogon-decider-sim/src/native.rs new file mode 100644 index 000000000..89836ed88 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/src/native.rs @@ -0,0 +1,212 @@ +//! A native runner for [`ScenarioIr`] scenarios, executing directly against a +//! [`trogon_decider::Decider`] implementation instead of a compiled wasm component. +//! +//! Envelope decoding is kept identical to the wasm guest bridge by reusing the exact generic +//! functions the `export_decider!` macro expands into +//! ([`trogon_decider_guest_sdk::decode_command`], [`trogon_decider_guest_sdk::evolve_one`], +//! [`trogon_decider_guest_sdk::decide_command`]): the same code runs on both the wasm and the +//! native side, not a hand-mirrored copy of its logic. + +use trogon_decider::{Decider, EventDecode, EventEncode, EventType}; +use trogon_decider_guest_sdk::{ + AnyEnvelopeParts, AnyEnvelopeView, CommandEnvelopeView, DecideErrorView, DomainErrorParts, decide_command, + decode_command, evolve_one, +}; + +use crate::ir::{ScenarioIr, StepOutcome, WireEnvelope}; + +impl CommandEnvelopeView for WireEnvelope { + fn command_type(&self) -> &str { + &self.type_url + } + + fn command_payload(&self) -> &[u8] { + &self.payload + } +} + +impl AnyEnvelopeView for WireEnvelope { + fn event_type(&self) -> &str { + &self.type_url + } + + fn event_payload(&self) -> &[u8] { + &self.payload + } +} + +impl From for WireEnvelope { + fn from(value: AnyEnvelopeParts) -> Self { + Self { + type_url: value.type_url, + payload: value.payload, + } + } +} + +/// Wire projection of a native decider's domain error. +/// +/// Mirrors the WIT `domain-error` record's `code`/`message`/`details`, so a native rejection or +/// fault carries the same shape as the wasm guest bridge's. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{code}: {message}")] +pub struct NativeDomainError { + /// The error's stable, machine-readable code. + pub code: String, + /// The error's human-readable message. + pub message: String, + /// The error's `#[source]` chain, flattened to ordered `(label, text)` pairs. + pub details: Vec<(String, String)>, +} + +impl From for NativeDomainError { + fn from(value: DomainErrorParts) -> Self { + Self { + code: value.code, + message: value.message, + details: value.details, + } + } +} + +/// A native `decide` call's failure, mirroring the WIT `decide-error` variant. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NativeDecideError { + /// The command was rejected by a business rule. + Rejected(NativeDomainError), + /// The command failed for a reason other than a business rule. + Faulted(NativeDomainError), +} + +impl DecideErrorView for NativeDecideError { + fn rejected(parts: DomainErrorParts) -> Self { + Self::Rejected(parts.into()) + } + + fn faulted(parts: DomainErrorParts) -> Self { + Self::Faulted(parts.into()) + } +} + +/// Decodes a wire command envelope into a typed command. +/// +/// Reuses the exact contract the wasm guest bridge's `decode_command` enforces: the envelope's +/// type URL must match `expected_type_url`, its payload decodes via `Proto::decode_from_slice`, +/// then converts to `Cmd` via `Cmd::try_from(proto)`. +pub fn decode_native_command( + expected_type_url: &str, + envelope: &WireEnvelope, +) -> Result +where + Proto: buffa::Message, + Cmd: TryFrom, +{ + decode_command::(expected_type_url, envelope.clone()) +} + +/// Folds one wire event envelope into a decider's state. +/// +/// Reuses the exact contract the wasm guest bridge's `evolve_one` enforces: an envelope outside +/// `C::Event`'s decoded set is skipped rather than treated as an error, mirroring native runtime +/// replay. +pub fn native_evolve_one(state: C::State, envelope: &WireEnvelope) -> Result +where + C: Decider, + C::Event: EventDecode, +{ + evolve_one::(state, envelope.clone()) +} + +/// Decides a typed command against a decider's state. +/// +/// Reuses the exact contract the wasm guest bridge's `decide_command` enforces: `decide` and +/// `evolve` run through the same `evaluate_decision` pipeline the native runtime uses, and each +/// resulting event is encoded through `EventEncode`/`EventType`. +pub fn native_decide(command: &C, state: &C::State) -> Result, NativeDecideError> +where + C: Decider, + C::Event: EventEncode + EventType, +{ + decide_command::(command, state) +} + +/// A native decider dispatch table, mirroring one `export_decider!` bundle: several commands +/// sharing one `State`, dispatched on a wire command envelope's type URL exactly the way the +/// wasm guest's generated `decide`/`evolve`/`stream_id` exports do. +pub trait NativeDeciderBundle { + /// The bundle's shared decider state. + type State: Clone; + + /// Returns the state before any events have been replayed. + fn initial_state() -> Self::State; + + /// Resolves the stream id a command envelope targets. + fn stream_id(command: &WireEnvelope) -> Result; + + /// Folds one wire event envelope into the bundle's state. + fn evolve(state: Self::State, event: &WireEnvelope) -> Result; + + /// Decides a wire command envelope against the bundle's state. + fn decide(command: &WireEnvelope, state: &Self::State) -> Result, NativeDecideError>; +} + +/// Failure replaying a [`ScenarioIr`]'s `given` history or a step's forwarded events against a +/// [`NativeDeciderBundle`], before that step's outcome could be captured. +#[derive(Debug, thiserror::Error)] +pub enum NativeRunError { + /// Replaying the scenario's `given` history failed at the event with this index. + #[error("failed to replay given event {index}: {source}")] + Given { + /// The zero-based index of the failing `given` event. + index: usize, + /// The domain error the failing evolve produced. + #[source] + source: NativeDomainError, + }, + /// Folding step `index`'s emitted events into state before the next step failed. + #[error("failed to fold step {index}'s emitted events: {source}")] + Fold { + /// The zero-based index of the step whose emitted events failed to fold. + index: usize, + /// The domain error the failing evolve produced. + #[source] + source: NativeDomainError, + }, +} + +/// Runs a [`ScenarioIr`] against a [`NativeDeciderBundle`], capturing each step's raw +/// [`StepOutcome`]. +/// +/// Mirrors [`ScenarioIr::run_wasm`]'s session shape: `given` is replayed once, then each step's +/// command is decided against the accumulated state and its emitted events (if any) are folded +/// in before the next step, all without asserting the step's declared expectation. +pub fn run_native(scenario: &ScenarioIr) -> Result, NativeRunError> { + let mut state = N::initial_state(); + for (index, event) in scenario.given.iter().enumerate() { + state = N::evolve(state, event).map_err(|source| NativeRunError::Given { index, source })?; + } + + let mut outcomes = Vec::with_capacity(scenario.steps.len()); + for (index, step) in scenario.steps.iter().enumerate() { + match N::decide(&step.when, &state) { + Ok(events) => { + for event in &events { + state = N::evolve(state, event).map_err(|source| NativeRunError::Fold { index, source })?; + } + outcomes.push(StepOutcome::Events(events)); + } + Err(NativeDecideError::Rejected(error)) => outcomes.push(StepOutcome::Rejected { + code: error.code, + message: error.message, + }), + Err(NativeDecideError::Faulted(error)) => outcomes.push(StepOutcome::Faulted { + code: error.code, + message: error.message, + }), + } + } + Ok(outcomes) +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-sim/src/native/tests.rs b/rsworkspace/crates/trogon-decider-sim/src/native/tests.rs new file mode 100644 index 000000000..fc98f669c --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/src/native/tests.rs @@ -0,0 +1,200 @@ +use trogon_decider::{Decider, Decision, EventData, EventDecode, EventDecodeOutcome, EventEncode, EventType}; +use trogon_decider_guest_sdk::DomainErrorParts; + +use super::*; +use crate::ir::{ExpectedOutcome, ScenarioStep}; + +#[test] +fn native_domain_error_displays_code_and_message() { + let error = NativeDomainError::from(DomainErrorParts { + code: "already-exists".to_string(), + message: "schedule already exists".to_string(), + details: Vec::new(), + }); + assert_eq!(error.to_string(), "already-exists: schedule already exists"); +} + +fn domain_error_parts() -> DomainErrorParts { + DomainErrorParts { + code: "rejected".to_string(), + message: "nope".to_string(), + details: Vec::new(), + } +} + +#[test] +fn native_decide_error_view_routes_rejected_and_faulted() { + assert!(matches!( + NativeDecideError::rejected(domain_error_parts()), + NativeDecideError::Rejected(_) + )); + assert!(matches!( + NativeDecideError::faulted(domain_error_parts()), + NativeDecideError::Faulted(_) + )); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CounterEvent { + amount: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("counter codec error")] +struct CounterCodecError; + +impl EventEncode for CounterEvent { + type Error = CounterCodecError; + + fn encode(&self) -> Result, Self::Error> { + Ok(self.amount.to_le_bytes().to_vec()) + } +} + +impl EventType for CounterEvent { + type Error = CounterCodecError; + + fn event_type(&self) -> Result<&'static str, Self::Error> { + Ok("counter.Incremented") + } +} + +impl EventDecode for CounterEvent { + type Error = CounterCodecError; + + fn decode(event: EventData<'_>) -> Result, Self::Error> { + if event.event_type != "counter.Incremented" { + return Ok(EventDecodeOutcome::Skipped); + } + let bytes: [u8; 4] = event.payload.try_into().map_err(|_| CounterCodecError)?; + Ok(EventDecodeOutcome::Decoded(CounterEvent { + amount: u32::from_le_bytes(bytes), + })) + } +} + +struct Increment { + amount: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +enum CounterError { + #[error("increment would exceed the counter ceiling")] + TooLarge, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("invalid counter event")] +struct CounterEvolveError; + +impl Decider for Increment { + type StreamId = str; + type State = i64; + type Event = CounterEvent; + type DecideError = CounterError; + type EvolveError = CounterEvolveError; + + fn stream_id(&self) -> &Self::StreamId { + "counter" + } + + fn initial_state() -> Self::State { + 0 + } + + fn evolve(state: Self::State, event: &Self::Event) -> Result { + Ok(state + i64::from(event.amount)) + } + + fn decide(state: &Self::State, command: &Self) -> Result, Self::DecideError> { + if state + i64::from(command.amount) > 100 { + return Err(CounterError::TooLarge); + } + Ok(Decision::event(CounterEvent { amount: command.amount })) + } +} + +fn increment_envelope(amount: u32) -> WireEnvelope { + WireEnvelope::new("counter.Increment", amount.to_le_bytes().to_vec()) +} + +fn incremented_envelope(amount: u32) -> WireEnvelope { + WireEnvelope::new("counter.Incremented", amount.to_le_bytes().to_vec()) +} + +struct CounterBundle; + +impl NativeDeciderBundle for CounterBundle { + type State = i64; + + fn initial_state() -> Self::State { + Increment::initial_state() + } + + fn stream_id(_command: &WireEnvelope) -> Result { + Ok("counter".to_string()) + } + + fn evolve(state: Self::State, event: &WireEnvelope) -> Result { + native_evolve_one::(state, event) + } + + fn decide(command: &WireEnvelope, state: &Self::State) -> Result, NativeDecideError> { + let bytes: [u8; 4] = command + .payload + .clone() + .try_into() + .expect("test fixture always encodes a 4-byte amount"); + let amount = u32::from_le_bytes(bytes); + native_decide(&Increment { amount }, state) + } +} + +#[test] +fn run_native_accepts_and_folds_events_across_steps() { + let mut scenario = ScenarioIr::new("increment twice"); + scenario.steps.push(ScenarioStep { + when: increment_envelope(10), + expect: ExpectedOutcome::Accepted, + }); + scenario.steps.push(ScenarioStep { + when: increment_envelope(20), + expect: ExpectedOutcome::Accepted, + }); + + let outcomes = run_native::(&scenario).expect("scenario runs"); + assert_eq!(outcomes.len(), 2); + assert_eq!(outcomes[0], StepOutcome::Events(vec![incremented_envelope(10)])); + assert_eq!(outcomes[1], StepOutcome::Events(vec![incremented_envelope(20)])); +} + +#[test] +fn run_native_reports_rejection_once_ceiling_is_exceeded() { + let mut scenario = ScenarioIr::new("increment past the ceiling"); + scenario.steps.push(ScenarioStep { + when: increment_envelope(90), + expect: ExpectedOutcome::Accepted, + }); + scenario.steps.push(ScenarioStep { + when: increment_envelope(90), + expect: ExpectedOutcome::Rejected, + }); + + let outcomes = run_native::(&scenario).expect("scenario runs"); + assert_eq!(outcomes.len(), 2); + assert!(matches!(&outcomes[1], StepOutcome::Rejected { code, .. } if code == "rejected")); +} + +#[test] +fn run_native_replays_given_history_before_the_first_step() { + let mut scenario = ScenarioIr::new("given seeds state"); + scenario.given.push(incremented_envelope(50)); + scenario.steps.push(ScenarioStep { + when: increment_envelope(60), + expect: ExpectedOutcome::Rejected, + }); + + let outcomes = run_native::(&scenario).expect("scenario runs"); + assert_eq!(outcomes.len(), 1); + assert!(matches!(&outcomes[0], StepOutcome::Rejected { .. })); +} diff --git a/rsworkspace/crates/trogon-decider-sim/src/parity.rs b/rsworkspace/crates/trogon-decider-sim/src/parity.rs new file mode 100644 index 000000000..677b65069 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/src/parity.rs @@ -0,0 +1,108 @@ +//! Runs a [`ScenarioIr`] scenario through both a native decider bundle and a compiled wasm +//! component, asserting the two produce identical outcomes. +//! +//! This is the piece that turns "the same behavior is specified twice" into a test that fails +//! on divergence: without it, a codec bug, a codegen bug, or a WIT regression that only breaks +//! one of the two paths surfaces as silent drift rather than a failing assertion. + +use crate::host::SimInstance; +use crate::ir::{ScenarioIr, StepOutcome}; +use crate::native::{NativeDeciderBundle, NativeRunError, run_native}; +use crate::scenario::ScenarioError; + +/// A scenario's native and wasm runs disagreed, or one of them failed to run at all. +#[derive(Debug, thiserror::Error)] +pub enum ParityError { + /// The native runner failed before it could produce any step outcomes. + #[error("{scenario}: native run failed: {source}")] + Native { + /// The scenario's name. + scenario: String, + /// The native runner's underlying failure. + #[source] + source: NativeRunError, + }, + /// The wasm runner failed before it could produce any step outcomes. + #[error("{scenario}: wasm run failed: {source}")] + Wasm { + /// The scenario's name. + scenario: String, + /// The wasm runner's underlying failure. + #[source] + source: Box, + }, + /// The native and wasm runners produced a different number of step outcomes. + #[error("{scenario}: native produced {native_len} step outcome(s), wasm produced {wasm_len}")] + StepCountMismatch { + /// The scenario's name. + scenario: String, + /// The number of step outcomes the native runner produced. + native_len: usize, + /// The number of step outcomes the wasm runner produced. + wasm_len: usize, + }, + /// The native and wasm runners produced different outcomes for the same step. + #[error("{scenario}: step {index} outcome mismatch: native={native:?}, wasm={wasm:?}")] + Mismatch { + /// The scenario's name. + scenario: String, + /// The zero-based index of the mismatched step. + index: usize, + /// The native runner's outcome for this step. + native: Box, + /// The wasm runner's outcome for this step. + wasm: Box, + }, +} + +/// Runs `scenario` through both a native [`NativeDeciderBundle`] and a wasm component instance, +/// returning an error on the first divergence between the two runners' outcomes. +/// +/// A codec, codegen, or WIT-level divergence between the native decider and the compiled wasm +/// component makes this return `Err`, rather than each runner separately reporting success +/// against its own declared expectations. +pub fn assert_parity( + scenario: &ScenarioIr, + instance: &mut SimInstance, +) -> Result<(), ParityError> { + let native = run_native::(scenario).map_err(|source| ParityError::Native { + scenario: scenario.name.clone(), + source, + })?; + let wasm = scenario.run_wasm(instance).map_err(|source| ParityError::Wasm { + scenario: scenario.name.clone(), + source: Box::new(source), + })?; + + compare_outcomes(&scenario.name, native, wasm) +} + +/// Compares a native and a wasm run's step outcomes, independent of how each was produced. +/// +/// Kept separate from [`assert_parity`] so the comparison itself (step count, then per-step +/// equality) can be exercised without a real native decider or wasm component instance. +fn compare_outcomes(scenario_name: &str, native: Vec, wasm: Vec) -> Result<(), ParityError> { + if native.len() != wasm.len() { + return Err(ParityError::StepCountMismatch { + scenario: scenario_name.to_string(), + native_len: native.len(), + wasm_len: wasm.len(), + }); + } + + for (index, (native_outcome, wasm_outcome)) in native.iter().zip(wasm.iter()).enumerate() { + if native_outcome != wasm_outcome { + return Err(ParityError::Mismatch { + scenario: scenario_name.to_string(), + index, + native: Box::new(native_outcome.clone()), + wasm: Box::new(wasm_outcome.clone()), + }); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider-sim/src/parity/tests.rs b/rsworkspace/crates/trogon-decider-sim/src/parity/tests.rs new file mode 100644 index 000000000..f267fa87e --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/src/parity/tests.rs @@ -0,0 +1,50 @@ +use super::*; +use crate::ir::WireEnvelope; + +fn events_outcome(type_url: &str) -> StepOutcome { + StepOutcome::Events(vec![WireEnvelope::new(type_url, Vec::new())]) +} + +#[test] +fn compare_outcomes_passes_when_identical() { + let native = vec![events_outcome("a")]; + let wasm = vec![events_outcome("a")]; + compare_outcomes("scenario", native, wasm).expect("identical outcomes match"); +} + +#[test] +fn compare_outcomes_reports_step_count_mismatch() { + let native = vec![events_outcome("a")]; + let wasm = vec![events_outcome("a"), events_outcome("b")]; + let error = compare_outcomes("scenario", native, wasm).unwrap_err(); + assert!(matches!( + error, + ParityError::StepCountMismatch { + native_len: 1, + wasm_len: 2, + .. + } + )); +} + +#[test] +fn compare_outcomes_reports_the_first_mismatched_step() { + let native = vec![events_outcome("a"), events_outcome("b")]; + let wasm = vec![events_outcome("a"), events_outcome("different")]; + let error = compare_outcomes("scenario", native, wasm).unwrap_err(); + assert!(matches!(error, ParityError::Mismatch { index: 1, .. })); +} + +#[test] +fn compare_outcomes_distinguishes_rejected_from_faulted() { + let native = vec![StepOutcome::Rejected { + code: "already-exists".to_string(), + message: "nope".to_string(), + }]; + let wasm = vec![StepOutcome::Faulted { + code: "already-exists".to_string(), + message: "nope".to_string(), + }]; + let error = compare_outcomes("scenario", native, wasm).unwrap_err(); + assert!(matches!(error, ParityError::Mismatch { index: 0, .. })); +} diff --git a/rsworkspace/crates/trogon-decider-sim/src/scenario.rs b/rsworkspace/crates/trogon-decider-sim/src/scenario.rs index 65287903f..29611815d 100644 --- a/rsworkspace/crates/trogon-decider-sim/src/scenario.rs +++ b/rsworkspace/crates/trogon-decider-sim/src/scenario.rs @@ -7,138 +7,279 @@ use crate::host::SimInstance; /// or the domain error's code/message) instead of flattening it into a string. #[derive(Debug, thiserror::Error)] pub enum ScenarioError { + /// `.run(...)` was called without ever setting a command via `.when(...)`. #[error("scenario missing .when(...)")] MissingWhen, + /// A step's command was set via `.when(...)` but no `.then_*(...)` expectation followed it. #[error("scenario missing .then_events(...) or .then_rejected()")] MissingExpectation, + /// Opening the guest session before the first step failed. #[error("failed to open session")] OpenSession { + /// The underlying wasmtime error. #[source] source: wasmtime::Error, }, + /// Folding `given` or a step's forwarded events via `evolve` failed at the host-call level. #[error("failed to call evolve")] EvolveCall { + /// The underlying wasmtime error. #[source] source: wasmtime::Error, }, + /// Deciding a step's command failed at the host-call level. #[error("failed to call decide")] DecideCall { + /// The underlying wasmtime error. #[source] source: wasmtime::Error, }, - #[error("evolve failed: {code} — {message}")] - Evolve { code: String, message: String }, - #[error("expected error '{expected}', got rejection: {code} — {message}")] + /// Folding `given` or a step's forwarded events was rejected by the decider's domain logic. + #[error("evolve failed: {code}: {message}")] + Evolve { + /// The domain error's stable code. + code: String, + /// The domain error's human-readable message. + message: String, + }, + /// `.then_error(...)` expected a specific error but the command was rejected with a + /// different code or message. + #[error("expected error '{expected}', got rejection: {code}: {message}")] ErrorGotRejection { + /// The error code or message `.then_error(...)` expected. expected: String, + /// The rejection's actual code. code: String, + /// The rejection's actual message. message: String, }, - #[error("expected error '{expected}', got fault: {code} — {message}")] + /// `.then_error(...)` expected a specific error but the command faulted with a different + /// code or message. + #[error("expected error '{expected}', got fault: {code}: {message}")] ErrorGotFault { + /// The error code or message `.then_error(...)` expected. expected: String, + /// The fault's actual code. code: String, + /// The fault's actual message. message: String, }, + /// `.then_error(...)` expected the command to error but it was accepted instead. #[error("expected error '{expected}', got {count} event(s)")] - ErrorGotEvents { expected: String, count: usize }, + ErrorGotEvents { + /// The error code or message `.then_error(...)` expected. + expected: String, + /// The number of events the command was accepted with instead. + count: usize, + }, + /// `.then_rejected()` expected the command to be rejected but it was accepted instead. #[error("expected rejection, got {count} event(s)")] - RejectionGotEvents { count: usize }, - #[error("expected rejection, got fault: {code} — {message}")] - RejectionGotFault { code: String, message: String }, - #[error("expected acceptance, got rejection: {code} — {message}")] - AcceptanceGotRejection { code: String, message: String }, - #[error("expected acceptance, got fault: {code} — {message}")] - AcceptanceGotFault { code: String, message: String }, - #[error("rejected: {code} — {message}")] - EventsGotRejection { code: String, message: String }, - #[error("faulted: {code} — {message}")] - EventsGotFault { code: String, message: String }, + RejectionGotEvents { + /// The number of events the command was accepted with instead. + count: usize, + }, + /// `.then_rejected()` expected a rejection but the command faulted instead. + #[error("expected rejection, got fault: {code}: {message}")] + RejectionGotFault { + /// The fault's actual code. + code: String, + /// The fault's actual message. + message: String, + }, + /// `.then_accepted()` expected the command to be accepted but it was rejected instead. + #[error("expected acceptance, got rejection: {code}: {message}")] + AcceptanceGotRejection { + /// The rejection's actual code. + code: String, + /// The rejection's actual message. + message: String, + }, + /// `.then_accepted()` expected the command to be accepted but it faulted instead. + #[error("expected acceptance, got fault: {code}: {message}")] + AcceptanceGotFault { + /// The fault's actual code. + code: String, + /// The fault's actual message. + message: String, + }, + /// `.then_events(...)` expected specific events but the command was rejected instead. + #[error("rejected: {code}: {message}")] + EventsGotRejection { + /// The rejection's actual code. + code: String, + /// The rejection's actual message. + message: String, + }, + /// `.then_events(...)` expected specific events but the command faulted instead. + #[error("faulted: {code}: {message}")] + EventsGotFault { + /// The fault's actual code. + code: String, + /// The fault's actual message. + message: String, + }, + /// `.then_events(...)` expected a different number of events than the command produced. #[error("expected {expected} event(s), got {actual}")] - EventCountMismatch { expected: usize, actual: usize }, + EventCountMismatch { + /// The number of events `.then_events(...)` expected. + expected: usize, + /// The number of events the command actually produced. + actual: usize, + }, + /// `.then_events(...)` expected a specific event at `index` but the command produced a + /// different type or payload there. #[error( "event {index} mismatch: got type={got_type} payload={got_payload:?}, want type={want_type} payload={want_payload:?}" )] EventMismatch { + /// The zero-based index of the mismatched event. index: usize, + /// The type URL of the event actually produced. got_type: String, + /// The payload of the event actually produced. got_payload: Vec, + /// The type URL `.then_events(...)` expected at this index. want_type: String, + /// The payload `.then_events(...)` expected at this index. want_payload: Vec, }, + /// A step in a multi-step scenario failed. `index` is the zero-based + /// position of the failing step in the scenario's ordered step sequence. + /// + /// Only used when a scenario has more than one step; a scenario with + /// exactly one step surfaces that step's failure unwrapped, unchanged + /// from a scenario built with a single `.when(...)`/`.then_*(...)` pair. + #[error("step {index}: {source}")] + Step { + /// The zero-based index of the failing step. + index: usize, + /// The failing step's own error. + #[source] + source: Box, + }, +} + +/// The asserted outcome of a step's `decide` call. +#[derive(Debug)] +enum Expectation { + Events(Vec), + Rejected, + Accepted, + Error(String), +} + +/// One `when`/`then` pair in a scenario's ordered step sequence. +struct ScenarioStep { + when: host::CommandEnvelope, + expectation: Expectation, +} + +/// The step currently being built by the fluent API, before it is known +/// whether another `.when(...)` will follow it or `.run(...)` will finalize it. +#[derive(Default)] +struct PendingStep { + when: Option, + expectation: Option, +} + +impl PendingStep { + /// Moves this pending step into `steps` once both halves are present, + /// leaving an incomplete pending step (missing `when` or `expectation`) + /// untouched so `run` can report it as [`ScenarioError::MissingWhen`] or + /// [`ScenarioError::MissingExpectation`]. + fn flush_into(&mut self, steps: &mut Vec) { + if self.when.is_none() || self.expectation.is_none() { + return; + } + if let (Some(when), Some(expectation)) = (self.when.take(), self.expectation.take()) { + steps.push(ScenarioStep { when, expectation }); + } + } } /// Fluent given/when/then helper over a loaded WASM decider component. +/// +/// A scenario runs an ordered sequence of one or more `when`/`then` steps +/// against a single open guest session, the way a real caller issuing several +/// commands in a row would: each step's emitted events are folded into the +/// session via `evolve` before the next step's command is decided, instead of +/// each step starting from a fresh session. #[must_use = "sim scenarios must be completed with .run()"] pub struct SimScenario { given: Vec, - when: Option, - expect_events: Option>, - expect_rejected: bool, - expect_accepted: bool, - expect_error: Option, + steps: Vec, + current: PendingStep, } impl SimScenario { + /// Creates an empty scenario with no seeded history and no steps. pub fn new() -> Self { Self { given: Vec::new(), - when: None, - expect_events: None, - expect_rejected: false, - expect_accepted: false, - expect_error: None, + steps: Vec::new(), + current: PendingStep::default(), } } + /// Seeds the session's history, replayed via `evolve` before the first + /// step's command is decided. pub fn given(mut self, events: impl IntoIterator) -> Self { self.given.extend(events); self } + /// Sets the command for a step. + /// + /// Calling this a second time after a preceding step already has both a + /// command and a `.then_*(...)` expectation completes that step and + /// starts the next one, building an ordered multi-step scenario. A single + /// `.when(...)` call followed by exactly one `.then_*(...)` call behaves + /// exactly as it always has. pub fn when(mut self, command: host::CommandEnvelope) -> Self { - self.when = Some(command); + self.current.flush_into(&mut self.steps); + self.current.when = Some(command); self } + /// Sets the current step's expectation to a specific ordered list of events, completing the + /// step once paired with a preceding `.when(...)`. pub fn then_events(mut self, events: impl IntoIterator) -> Self { - self.expect_events = Some(events.into_iter().collect()); - self.expect_rejected = false; - self.expect_accepted = false; - self.expect_error = None; + self.current.expectation = Some(Expectation::Events(events.into_iter().collect())); self } + /// Sets the current step's expectation to a rejection, without asserting its code or + /// message. pub fn then_rejected(mut self) -> Self { - self.expect_events = None; - self.expect_rejected = true; - self.expect_accepted = false; - self.expect_error = None; + self.current.expectation = Some(Expectation::Rejected); self } /// Expect the command to be accepted (decide returns events), without /// asserting which events. Used for scenarios that only assert "not rejected". pub fn then_accepted(mut self) -> Self { - self.expect_events = None; - self.expect_rejected = false; - self.expect_accepted = true; - self.expect_error = None; + self.current.expectation = Some(Expectation::Accepted); self } /// Expect the command to error (rejected or faulted) with the given code or /// message. Matches the decode/decide outcome's `code` or `message` exactly. pub fn then_error(mut self, expected: impl Into) -> Self { - self.expect_events = None; - self.expect_rejected = false; - self.expect_accepted = false; - self.expect_error = Some(expected.into()); + self.current.expectation = Some(Expectation::Error(expected.into())); self } - pub fn run(self, instance: &mut SimInstance) -> Result<(), ScenarioError> { - let command = self.when.ok_or(ScenarioError::MissingWhen)?; + /// Runs the scenario's ordered steps against `instance`'s guest session, asserting each + /// step's expectation in turn and folding its accepted events into the session before the + /// next step. + pub fn run(mut self, instance: &mut SimInstance) -> Result<(), ScenarioError> { + self.current.flush_into(&mut self.steps); + if self.current.when.is_some() { + return Err(ScenarioError::MissingExpectation); + } + if self.steps.is_empty() { + return Err(ScenarioError::MissingWhen); + } let mut session = instance .open_session(None) @@ -154,14 +295,68 @@ impl SimScenario { })?; } - let outcome = session - .decide(&command) - .map_err(|source| ScenarioError::DecideCall { source })?; + let wrap_per_step = self.steps.len() > 1; + let mut forwarded = Vec::new(); + + for (index, step) in self.steps.into_iter().enumerate() { + if !forwarded.is_empty() { + session + .evolve(&forwarded) + .map_err(|source| wrap_step(index, wrap_per_step, ScenarioError::EvolveCall { source }))? + .map_err(|err| { + wrap_step( + index, + wrap_per_step, + ScenarioError::Evolve { + code: err.code, + message: err.message, + }, + ) + })?; + forwarded.clear(); + } + + let outcome = session + .decide(&step.when) + .map_err(|source| wrap_step(index, wrap_per_step, ScenarioError::DecideCall { source }))?; + forwarded = + check_outcome(outcome, step.expectation).map_err(|error| wrap_step(index, wrap_per_step, error))?; + } + + Ok(()) + } +} + +impl Default for SimScenario { + fn default() -> Self { + Self::new() + } +} + +fn wrap_step(index: usize, wrap: bool, error: ScenarioError) -> ScenarioError { + if wrap { + ScenarioError::Step { + index, + source: Box::new(error), + } + } else { + error + } +} - if let Some(expected) = self.expect_error { +/// Checks one step's `decide` outcome against its expectation, returning the +/// events actually emitted so the caller can fold them into the next step. +fn check_outcome( + outcome: Result, host::DecideError>, + expectation: Expectation, +) -> Result, ScenarioError> { + match expectation { + Expectation::Error(expected) => { let matches = |err: &host::DomainError| err.code == expected || err.message == expected; match outcome { - Err(host::DecideError::Rejected(err)) | Err(host::DecideError::Faulted(err)) if matches(&err) => Ok(()), + Err(host::DecideError::Rejected(err)) | Err(host::DecideError::Faulted(err)) if matches(&err) => { + Ok(Vec::new()) + } Err(host::DecideError::Rejected(err)) => Err(ScenarioError::ErrorGotRejection { expected, code: err.code, @@ -177,29 +372,27 @@ impl SimScenario { count: events.len(), }), } - } else if self.expect_rejected { - match outcome { - Err(host::DecideError::Rejected(_)) => Ok(()), - Ok(events) => Err(ScenarioError::RejectionGotEvents { count: events.len() }), - Err(host::DecideError::Faulted(err)) => Err(ScenarioError::RejectionGotFault { - code: err.code, - message: err.message, - }), - } - } else if self.expect_accepted { - match outcome { - Ok(_) => Ok(()), - Err(host::DecideError::Rejected(err)) => Err(ScenarioError::AcceptanceGotRejection { - code: err.code, - message: err.message, - }), - Err(host::DecideError::Faulted(err)) => Err(ScenarioError::AcceptanceGotFault { - code: err.code, - message: err.message, - }), - } - } else { - let expected = self.expect_events.ok_or(ScenarioError::MissingExpectation)?; + } + Expectation::Rejected => match outcome { + Err(host::DecideError::Rejected(_)) => Ok(Vec::new()), + Ok(events) => Err(ScenarioError::RejectionGotEvents { count: events.len() }), + Err(host::DecideError::Faulted(err)) => Err(ScenarioError::RejectionGotFault { + code: err.code, + message: err.message, + }), + }, + Expectation::Accepted => match outcome { + Ok(events) => Ok(events), + Err(host::DecideError::Rejected(err)) => Err(ScenarioError::AcceptanceGotRejection { + code: err.code, + message: err.message, + }), + Err(host::DecideError::Faulted(err)) => Err(ScenarioError::AcceptanceGotFault { + code: err.code, + message: err.message, + }), + }, + Expectation::Events(expected) => { let actual = outcome.map_err(|err| match err { host::DecideError::Rejected(err) => ScenarioError::EventsGotRejection { code: err.code, @@ -227,17 +420,11 @@ impl SimScenario { }); } } - Ok(()) + Ok(actual) } } } -impl Default for SimScenario { - fn default() -> Self { - Self::new() - } -} - /// Compares two event envelopes by meaning rather than by raw wire bytes. /// /// Protobuf encoding is not canonical, so semantically identical events can diff --git a/rsworkspace/crates/trogon-decider-sim/src/session.rs b/rsworkspace/crates/trogon-decider-sim/src/session.rs index 9579c0aa2..0ffc2dde2 100644 --- a/rsworkspace/crates/trogon-decider-sim/src/session.rs +++ b/rsworkspace/crates/trogon-decider-sim/src/session.rs @@ -1,32 +1,58 @@ +use trogon_decider_wasm_runtime::WasmEngineConfig; use trogon_decider_wit::host::{self, Decider}; use wasmtime::Store; +use crate::host::{SimGuestState, arm_guest_call}; + /// A guest session resource with deterministic teardown via [`resource_drop`](host::drop_session). pub struct SimSession<'a, T: 'static> { pub(crate) bindings: &'a Decider, - pub(crate) store: &'a mut Store, + pub(crate) store: &'a mut Store>, + pub(crate) config: WasmEngineConfig, pub(crate) session: host::Session, } impl SimSession<'_, T> { + /// Arms the guest call budget and folds `events` into this session's decider state via the + /// guest's `evolve` export. pub fn evolve(&mut self, events: &[host::AnyEnvelope]) -> Result, wasmtime::Error> { + arm_guest_call(self.store, &self.config)?; host::evolve(self.bindings, self.store, self.session, events) } + /// Arms the guest call budget and decides `command` against this session's current state via + /// the guest's `decide` export. pub fn decide( &mut self, command: &host::CommandEnvelope, ) -> Result, host::DecideError>, wasmtime::Error> { + arm_guest_call(self.store, &self.config)?; host::decide(self.bindings, self.store, self.session, command) } + /// Arms the guest call budget and returns this session's current state snapshot, if the + /// decider supports snapshotting. pub fn snapshot(&mut self) -> Result>, wasmtime::Error> { + arm_guest_call(self.store, &self.config)?; host::snapshot(self.bindings, self.store, self.session) } } impl Drop for SimSession<'_, T> { fn drop(&mut self) { - let _ = host::drop_session(self.bindings, self.store, self.session); + // Mirrors production's execution.rs: `drop_session` runs without a + // fresh arm, reusing whatever fuel/epoch budget remains from the + // immediately preceding armed call. + if let Err(source) = host::drop_session(self.bindings, self.store, self.session) { + tracing::warn!(error = %source, "sim guest session teardown failed"); + #[cfg(any(test, feature = "test-support"))] + #[allow( + clippy::panic, + reason = "surfaces swallowed teardown failures loudly in tests/sim callers" + )] + if !std::thread::panicking() { + panic!("sim guest session teardown failed: {source}"); + } + } } } diff --git a/rsworkspace/crates/trogon-decider-sim/tests/parity.rs b/rsworkspace/crates/trogon-decider-sim/tests/parity.rs new file mode 100644 index 000000000..28497a39e --- /dev/null +++ b/rsworkspace/crates/trogon-decider-sim/tests/parity.rs @@ -0,0 +1,137 @@ +//! Parity between the scheduler schedules native decider and its compiled wasm component. +#![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] + +use std::collections::BTreeSet; +use std::fs; + +use trogon_decider::Decider; +use trogon_decider_sim::{ + NativeDecideError, NativeDeciderBundle, NativeDomainError, SimFixture, SimHost, WireEnvelope, assert_parity, + decode_native_command, native_decide, native_evolve_one, +}; +use trogon_decider_test::Suite; +use trogon_scheduler_domain::{CreateSchedule, PauseSchedule, RemoveSchedule, ResumeSchedule}; +use trogonai_proto::scheduler::schedules::{ + CREATE_SCHEDULE_TYPE_URL, PAUSE_SCHEDULE_TYPE_URL, REMOVE_SCHEDULE_TYPE_URL, RESUME_SCHEDULE_TYPE_URL, state_v1, v1, +}; + +fn invalid_command(other: &str) -> NativeDomainError { + NativeDomainError { + code: "invalid-command".to_string(), + message: format!("unknown command type '{other}'"), + details: Vec::new(), + } +} + +/// Mirrors `wasm-components/trogon-schedules-decider`'s `export_decider!` bundle: the same four +/// commands, dispatched by type URL, with `CreateSchedule` as the canonical command whose shared +/// `State`/`Event`/`EvolveError` folds every event regardless of which command produced it. +struct SchedulesBundle; + +impl NativeDeciderBundle for SchedulesBundle { + type State = state_v1::State; + + fn initial_state() -> Self::State { + trogon_scheduler_domain::initial_state() + } + + fn stream_id(command: &WireEnvelope) -> Result { + match command.type_url.as_str() { + CREATE_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(CREATE_SCHEDULE_TYPE_URL, command)?; + Ok(command.stream_id().to_string()) + } + PAUSE_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(PAUSE_SCHEDULE_TYPE_URL, command)?; + Ok(command.stream_id().to_string()) + } + REMOVE_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(REMOVE_SCHEDULE_TYPE_URL, command)?; + Ok(command.stream_id().to_string()) + } + RESUME_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(RESUME_SCHEDULE_TYPE_URL, command)?; + Ok(command.stream_id().to_string()) + } + other => Err(invalid_command(other)), + } + } + + fn evolve(state: Self::State, event: &WireEnvelope) -> Result { + native_evolve_one::(state, event) + } + + fn decide(command: &WireEnvelope, state: &Self::State) -> Result, NativeDecideError> { + match command.type_url.as_str() { + CREATE_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(CREATE_SCHEDULE_TYPE_URL, command) + .map_err(NativeDecideError::Faulted)?; + native_decide(&command, state) + } + PAUSE_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(PAUSE_SCHEDULE_TYPE_URL, command) + .map_err(NativeDecideError::Faulted)?; + native_decide(&command, state) + } + REMOVE_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(REMOVE_SCHEDULE_TYPE_URL, command) + .map_err(NativeDecideError::Faulted)?; + native_decide(&command, state) + } + RESUME_SCHEDULE_TYPE_URL => { + let command = + decode_native_command::(RESUME_SCHEDULE_TYPE_URL, command) + .map_err(NativeDecideError::Faulted)?; + native_decide(&command, state) + } + other => Err(NativeDecideError::Faulted(invalid_command(other))), + } + } +} + +fn schedules_suite() -> Suite { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../cli/trogon-decider-test/schedules.yaml" + ); + let yaml = fs::read_to_string(path).expect("read schedules.yaml"); + Suite::from_yaml(&yaml).expect("parse schedules.yaml") +} + +/// Every scenario in the CLI's `schedules.yaml`, converted through the same +/// `Suite`/`Scenario::to_ir` code path the `decider-test` binary uses, run through both the +/// native scheduler schedules decider and its compiled wasm component. A codec bug, a codegen +/// bug, or a WIT regression that only breaks one of the two paths fails this test even if each +/// runner separately satisfies the scenario's own declared expectations. +#[test] +fn schedules_yaml_scenarios_have_native_wasm_parity() { + let suite = schedules_suite(); + let scenarios = suite.to_ir().expect("schedules.yaml converts to ir"); + assert!(!scenarios.is_empty(), "schedules.yaml declares no scenarios"); + + let names: BTreeSet<&str> = scenarios.iter().map(|scenario| scenario.name.as_str()).collect(); + assert!( + names.contains("create, pause, then resume a schedule across one session"), + "expected the multi-step create/pause/resume scenario to still be declared" + ); + assert!( + names.iter().any(|name| name.starts_with("reject ")), + "expected at least one rejection scenario to still be declared" + ); + + let wasm = SimFixture::schedules().bytes().to_vec(); + let host = SimHost::load(&wasm).expect("load schedules wasm component"); + + for scenario in &scenarios { + let mut instance = host.instantiate(()).expect("instantiate schedules component"); + assert_parity::(scenario, &mut instance) + .unwrap_or_else(|error| panic!("scenario '{}' diverged: {error}", scenario.name)); + } +} diff --git a/rsworkspace/crates/trogon-decider-sim/tests/schedules.rs b/rsworkspace/crates/trogon-decider-sim/tests/schedules.rs index 998994108..344362319 100644 --- a/rsworkspace/crates/trogon-decider-sim/tests/schedules.rs +++ b/rsworkspace/crates/trogon-decider-sim/tests/schedules.rs @@ -4,10 +4,14 @@ use buffa::Message as _; use buffa::MessageField; use buffa::MessageName as _; -use trogon_decider_sim::{ScenarioError, SimFixture, SimHost, SimScenario, assert_zero_imports}; +use trogon_decider_sim::{ + ScenarioError, SimError, SimFixture, SimHost, SimScenario, WasmEngineConfig, assert_zero_imports, +}; use trogon_decider_wit::host::{self, CommandEnvelope}; use trogonai_proto::content::v1alpha1 as content_v1alpha1; -use trogonai_proto::scheduler::schedules::{CREATE_SCHEDULE_TYPE_URL, PAUSE_SCHEDULE_TYPE_URL, v1}; +use trogonai_proto::scheduler::schedules::{ + CREATE_SCHEDULE_TYPE_URL, PAUSE_SCHEDULE_TYPE_URL, RESUME_SCHEDULE_TYPE_URL, v1, +}; fn schedules_wasm() -> Vec { SimFixture::schedules().bytes().to_vec() @@ -108,6 +112,36 @@ fn pause_command(id: &str) -> CommandEnvelope { } } +fn resume_command(id: &str) -> CommandEnvelope { + CommandEnvelope { + type_: RESUME_SCHEDULE_TYPE_URL.to_string(), + payload: v1::ResumeSchedule { + schedule_id: id.to_string(), + } + .encode_to_vec(), + } +} + +fn schedule_paused_event(id: &str) -> host::AnyEnvelope { + host::AnyEnvelope { + type_: v1::SchedulePaused::FULL_NAME.to_string(), + payload: v1::SchedulePaused { + schedule_id: id.to_string(), + } + .encode_to_vec(), + } +} + +fn schedule_resumed_event(id: &str) -> host::AnyEnvelope { + host::AnyEnvelope { + type_: v1::ScheduleResumed::FULL_NAME.to_string(), + payload: v1::ScheduleResumed { + schedule_id: id.to_string(), + } + .encode_to_vec(), + } +} + fn unknown_command() -> CommandEnvelope { CommandEnvelope { type_: "type.googleapis.com/trogonai.scheduler.schedules.v1.DoesNotExist".to_string(), @@ -144,6 +178,34 @@ fn stream_id_returns_schedule_id() { assert_eq!(stream, "backup"); } +#[test] +fn sim_host_applies_the_production_engine_config_by_default() { + let host = SimHost::load(&schedules_wasm()).unwrap(); + assert_eq!(host.config(), WasmEngineConfig::default()); +} + +#[test] +fn a_runaway_decider_traps_on_fuel_exhaustion_like_production_would() { + let config = WasmEngineConfig::default().with_fuel_per_call(1); + let host = SimHost::load_with_config(&schedules_wasm(), config).unwrap(); + + // Every guest call is armed with this same starved fuel budget (mirroring + // production's `arm_guest_call`), so either instantiation or the first + // guest export call must trap with `OutOfFuel`, not run unbounded. + let trap = match host.instantiate(()) { + Err(SimError::Instantiate { source }) => source, + Err(other) => panic!("expected SimError::Instantiate on fuel exhaustion, got {other}"), + Ok(mut instance) => match instance.descriptor() { + Err(source) => source, + Ok(descriptor) => panic!("expected a fuel-exhaustion trap, but the guest call succeeded: {descriptor:?}"), + }, + }; + assert!( + matches!(trap.downcast_ref::(), Some(wasmtime::Trap::OutOfFuel)), + "expected an OutOfFuel trap, got {trap}" + ); +} + #[test] fn then_accepted_passes_for_create() { let host = SimHost::load(&schedules_wasm()).unwrap(); @@ -448,6 +510,49 @@ fn evolve_skips_events_outside_this_deciders_set() { .unwrap(); } +#[test] +fn multi_step_scenario_feeds_events_forward_within_one_session() { + let host = SimHost::load(&schedules_wasm()).unwrap(); + let mut instance = host.instantiate(()).unwrap(); + + // Each step's `then_events` output is folded into the same open session + // before the next step's command is decided, so resume only succeeds + // because it observes the pause this scenario decided one step earlier. + SimScenario::new() + .when(create_command("backup")) + .then_events([schedule_created_event("backup")]) + .when(pause_command("backup")) + .then_events([schedule_paused_event("backup")]) + .when(resume_command("backup")) + .then_events([schedule_resumed_event("backup")]) + .run(&mut instance) + .unwrap(); +} + +#[test] +fn multi_step_scenario_reports_which_step_failed() { + let host = SimHost::load(&schedules_wasm()).unwrap(); + let mut instance = host.instantiate(()).unwrap(); + + let error = SimScenario::new() + .when(create_command("backup")) + .then_events([schedule_created_event("backup")]) + .when(pause_command("backup")) + .then_rejected() + .run(&mut instance) + .unwrap_err(); + match &error { + ScenarioError::Step { index, source } => { + assert_eq!(*index, 1); + assert!(matches!( + source.as_ref(), + ScenarioError::RejectionGotEvents { count: 1 } + )); + } + other => panic!("expected ScenarioError::Step, got {other}"), + } +} + #[test] fn snapshot_round_trips_into_a_restored_session() { let host = SimHost::load(&schedules_wasm()).unwrap(); diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/Cargo.toml b/rsworkspace/crates/trogon-decider-wasm-runtime/Cargo.toml index d2baf3fdc..28c054bcc 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/Cargo.toml @@ -7,16 +7,27 @@ license = "Apache-2.0" repository = "https://github.com/TrogonStack/trogonai" publish = false -[lints] -workspace = true +[lints.rust] +missing_docs = "warn" +warnings = "deny" +unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(dylint_lib, values(any()))'] } + +[lints.clippy] +all = "deny" +expect_used = "deny" +panic = "deny" +unwrap_used = "deny" [dependencies] +opentelemetry = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } trogon-decider-runtime = { workspace = true } trogon-decider-wit = { workspace = true, features = ["host"] } +trogon-semconv = { workspace = true } trogon-std = { workspace = true, features = ["uuid"] } -wasmtime = { version = "=46.0.1", default-features = false, features = ["anyhow", "component-model", "runtime", "cranelift"] } +wasmtime = { version = "=46.0.1", default-features = false, features = ["anyhow", "component-model", "pooling-allocator", "runtime", "cranelift"] } [dev-dependencies] buffa = { workspace = true } diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/README.md b/rsworkspace/crates/trogon-decider-wasm-runtime/README.md new file mode 100644 index 000000000..e7f6585f6 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/README.md @@ -0,0 +1,21 @@ +# trogon-decider-wasm-runtime + +The `-runtime` suffix (per [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md)) marks +execution machinery, here for deciders compiled to WASM components rather than linked in +natively. `WasmDeciderModule::load` compiles a component, structurally enforces it declares +zero imports (by instantiating it against an empty `wasmtime::component::Linker`), and probes +its `descriptor()` once. `WasmDeciderEngine`/`WasmEngineConfig` configure one shared wasmtime +`Engine` per process with fuel metering and epoch-based interruption, giving every guest call a +bounded instruction count and a bounded wall-clock deadline. `WasmCommandExecution` is this +crate's analog of `trogon-decider-runtime`'s `CommandExecution`, driving the WIT `session` +resource through replay, decide, event-folding, and snapshot instead of a native `Decider` +implementation directly. `DeciderRegistry`/`DeciderRegistryHandle` route command types to +modules and support activating a new module version or retiring a route while executions are +in flight, with snapshot identity (`WasmSnapshotId`) keyed by module name and version so a +version swap never has to migrate or invalidate a prior version's snapshots. + +This crate reuses `trogon-decider-runtime`'s storage-neutral ports (`StreamRead`, +`StreamAppend`, `SnapshotRead`, `SnapshotWrite`) rather than depending on any specific backend. + +See [Decider Platform](../../../docs/architecture/decider.md) for the exact engine budget +values, the decide/fold/snapshot call ordering, and registry rollout semantics in full. diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail.rs index ccddd97c3..b4bc96438 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail.rs @@ -1,14 +1,20 @@ /// Domain failure reported by a guest decider across the WIT boundary. /// /// This mirrors the WIT `domain-error` record verbatim: a stable machine -/// `code` and a human-readable `message`. It carries no source error because -/// the guest error already crossed a serialization boundary; there is nothing -/// further to preserve on the host side. +/// `code`, a human-readable `message`, and the `details` pairs the guest +/// attached (the bridge populates these with the causal chain below the +/// top-level message). It carries no source error because the guest error +/// already crossed a serialization boundary; the `details` pairs are all +/// that survives of the original chain. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[error("{code}: {message}")] pub struct DomainErrorDetail { + /// Stable, machine-readable rejection code the guest reported. pub code: String, + /// Human-readable description of the failure. pub message: String, + /// Ordered key/value pairs carrying the error's source chain, most specific cause last. + pub details: Vec<(String, String)>, } impl From for DomainErrorDetail { @@ -16,6 +22,7 @@ impl From for DomainErrorDetail { Self { code: value.code, message: value.message, + details: value.details, } } } diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail/tests.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail/tests.rs index 3d868de88..d6c1f90dd 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail/tests.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/domain_error_detail/tests.rs @@ -5,6 +5,7 @@ fn displays_as_code_colon_message() { let detail = DomainErrorDetail { code: "already-registered".to_string(), message: "schedule already exists".to_string(), + details: Vec::new(), }; assert_eq!(detail.to_string(), "already-registered: schedule already exists"); } @@ -14,6 +15,7 @@ fn implements_error_trait_without_source() { let detail = DomainErrorDetail { code: "code".to_string(), message: "message".to_string(), + details: Vec::new(), }; assert!(std::error::Error::source(&detail).is_none()); } @@ -23,8 +25,10 @@ fn converts_from_wit_domain_error() { let wit_error = trogon_decider_wit::host::DomainError { code: "code".to_string(), message: "message".to_string(), + details: vec![("cause".to_string(), "inner failure".to_string())], }; let detail = DomainErrorDetail::from(wit_error); assert_eq!(detail.code, "code"); assert_eq!(detail.message, "message"); + assert_eq!(detail.details, vec![("cause".to_string(), "inner failure".to_string())]); } diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine.rs index f1cb12bd5..04492dc39 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine.rs @@ -1,4 +1,8 @@ -use wasmtime::{Engine, Store, StoreLimits, StoreLimitsBuilder}; +use std::time::Duration; + +use wasmtime::{ + Engine, EngineWeak, InstanceAllocationStrategy, PoolingAllocationConfig, Store, StoreLimits, StoreLimitsBuilder, +}; /// Default fuel budget for a single guest export call (`decide`, `evolve`, or /// `snapshot`). Wasmtime decrements fuel per instruction executed, so a guest @@ -10,12 +14,74 @@ pub const DEFAULT_FUEL_PER_CALL: u64 = 10_000_000; /// memory one guest session can pin regardless of how many events it replays. pub const DEFAULT_MAX_MEMORY_BYTES: usize = 64 * 1024 * 1024; +/// Default ceiling on the element count of any table a guest session's store +/// may create. +/// +/// Wasmtime otherwise leaves `table.grow` unbounded. Inspecting the compiled +/// schedules decider component with `wasm-tools print` shows its largest +/// table (the guest module's indirect-call table) is fixed at 270 elements; +/// this ceiling leaves generous headroom for larger deciders while still +/// rejecting a guest that tries to grow a table without bound. +pub const DEFAULT_MAX_TABLE_ELEMENTS: usize = 8_192; + +/// Default ceiling on how many core wasm module instantiations a single +/// guest session's store may create. +/// +/// Every component this crate loads implements the fixed +/// `trogon:decider@0.1.0` world, which declares exactly one resource type +/// (`session`). Inspecting the compiled schedules decider component with +/// `wasm-tools print` shows that shape always instantiates 3 real core +/// modules: the guest's own module plus a pair of tiny resource-destructor +/// shim modules `cargo component` emits to break a circular dependency +/// between the guest module and its destructor. That structure is invariant +/// across every decider this crate loads. This ceiling leaves headroom above +/// the measured minimum of 3 without reopening wasmtime's own default of +/// 10,000. +pub const DEFAULT_MAX_INSTANCES_PER_SESSION: usize = 8; + +/// Default ceiling on how many wasm tables a single guest session's store may +/// create. The schedules decider fixture allocates 2 (the main module's +/// indirect-call table and the destructor shim's 1-element table); see +/// [`DEFAULT_MAX_INSTANCES_PER_SESSION`] for how this was measured. +pub const DEFAULT_MAX_TABLES_PER_SESSION: usize = 6; + +/// Default ceiling on how many wasm linear memories a single guest session's +/// store may create. The schedules decider fixture allocates exactly 1 (only +/// the main module defines a memory; the destructor shims do not); see +/// [`DEFAULT_MAX_INSTANCES_PER_SESSION`] for how this was measured. +pub const DEFAULT_MAX_MEMORIES_PER_SESSION: usize = 4; + +/// Default ceiling on how many guest sessions the pooling allocator keeps +/// warm allocation slots for concurrently. +/// +/// The pooling allocator's aggregate totals are sized as this many sessions' +/// worth of [`DEFAULT_MAX_INSTANCES_PER_SESSION`] core instances, +/// [`DEFAULT_MAX_TABLES_PER_SESSION`] tables, and +/// [`DEFAULT_MAX_MEMORIES_PER_SESSION`] memories. +pub const DEFAULT_MAX_CONCURRENT_SESSIONS: usize = 256; + +/// Default cadence at which the engine's background ticker increments +/// wasmtime's epoch counter for epoch-based interruption. +pub const DEFAULT_EPOCH_TICK_INTERVAL: Duration = Duration::from_millis(50); + +/// Default number of epoch ticks a single guest export call is allowed +/// before it is interrupted. At the default [`DEFAULT_EPOCH_TICK_INTERVAL`] +/// cadence this is a 2 second wall-clock budget per call. +pub const DEFAULT_EPOCH_TICKS_PER_CALL: u64 = 40; + /// Tunable resource limits applied to every guest session created from a /// [`WasmDeciderEngine`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct WasmEngineConfig { max_memory_bytes: usize, fuel_per_call: u64, + max_table_elements: usize, + max_instances_per_session: usize, + max_tables_per_session: usize, + max_memories_per_session: usize, + max_concurrent_sessions: usize, + epoch_tick_interval: Duration, + epoch_ticks_per_call: u64, } impl WasmEngineConfig { @@ -29,6 +95,45 @@ impl WasmEngineConfig { self.fuel_per_call } + /// Returns the table element ceiling applied to each guest store. + pub fn max_table_elements(&self) -> usize { + self.max_table_elements + } + + /// Returns the core module instantiation ceiling applied to each guest + /// store. + pub fn max_instances_per_session(&self) -> usize { + self.max_instances_per_session + } + + /// Returns the table ceiling applied to each guest store. + pub fn max_tables_per_session(&self) -> usize { + self.max_tables_per_session + } + + /// Returns the linear memory ceiling applied to each guest store. + pub fn max_memories_per_session(&self) -> usize { + self.max_memories_per_session + } + + /// Returns how many guest sessions the pooling allocator keeps warm + /// allocation slots for concurrently. + pub fn max_concurrent_sessions(&self) -> usize { + self.max_concurrent_sessions + } + + /// Returns the cadence at which the background ticker increments the + /// wasmtime epoch. + pub fn epoch_tick_interval(&self) -> Duration { + self.epoch_tick_interval + } + + /// Returns the epoch ticks allowed for a single guest export call before + /// it is interrupted. + pub fn epoch_ticks_per_call(&self) -> u64 { + self.epoch_ticks_per_call + } + /// Overrides the memory ceiling applied to each guest store. pub fn with_max_memory_bytes(mut self, max_memory_bytes: usize) -> Self { self.max_memory_bytes = max_memory_bytes; @@ -40,6 +145,52 @@ impl WasmEngineConfig { self.fuel_per_call = fuel_per_call; self } + + /// Overrides the table element ceiling applied to each guest store. + pub fn with_max_table_elements(mut self, max_table_elements: usize) -> Self { + self.max_table_elements = max_table_elements; + self + } + + /// Overrides the core module instantiation ceiling applied to each guest + /// store. + pub fn with_max_instances_per_session(mut self, max_instances_per_session: usize) -> Self { + self.max_instances_per_session = max_instances_per_session; + self + } + + /// Overrides the table ceiling applied to each guest store. + pub fn with_max_tables_per_session(mut self, max_tables_per_session: usize) -> Self { + self.max_tables_per_session = max_tables_per_session; + self + } + + /// Overrides the linear memory ceiling applied to each guest store. + pub fn with_max_memories_per_session(mut self, max_memories_per_session: usize) -> Self { + self.max_memories_per_session = max_memories_per_session; + self + } + + /// Overrides how many guest sessions the pooling allocator keeps warm + /// allocation slots for concurrently. + pub fn with_max_concurrent_sessions(mut self, max_concurrent_sessions: usize) -> Self { + self.max_concurrent_sessions = max_concurrent_sessions; + self + } + + /// Overrides the cadence at which the background ticker increments the + /// wasmtime epoch. + pub fn with_epoch_tick_interval(mut self, epoch_tick_interval: Duration) -> Self { + self.epoch_tick_interval = epoch_tick_interval; + self + } + + /// Overrides the epoch ticks allowed for a single guest export call + /// before it is interrupted. + pub fn with_epoch_ticks_per_call(mut self, epoch_ticks_per_call: u64) -> Self { + self.epoch_ticks_per_call = epoch_ticks_per_call; + self + } } impl Default for WasmEngineConfig { @@ -47,12 +198,20 @@ impl Default for WasmEngineConfig { Self { max_memory_bytes: DEFAULT_MAX_MEMORY_BYTES, fuel_per_call: DEFAULT_FUEL_PER_CALL, + max_table_elements: DEFAULT_MAX_TABLE_ELEMENTS, + max_instances_per_session: DEFAULT_MAX_INSTANCES_PER_SESSION, + max_tables_per_session: DEFAULT_MAX_TABLES_PER_SESSION, + max_memories_per_session: DEFAULT_MAX_MEMORIES_PER_SESSION, + max_concurrent_sessions: DEFAULT_MAX_CONCURRENT_SESSIONS, + epoch_tick_interval: DEFAULT_EPOCH_TICK_INTERVAL, + epoch_ticks_per_call: DEFAULT_EPOCH_TICKS_PER_CALL, } } } /// Per-session store state. Wraps [`StoreLimits`] so wasmtime enforces the -/// memory ceiling on `memory.grow` without the host having to poll it. +/// resource ceilings on growth (memory, table elements, instance, table, and +/// memory counts) without the host having to poll them. pub(crate) struct GuestState { pub(crate) limits: StoreLimits, } @@ -60,17 +219,27 @@ pub(crate) struct GuestState { impl GuestState { fn new(config: &WasmEngineConfig) -> Self { Self { - limits: StoreLimitsBuilder::new().memory_size(config.max_memory_bytes).build(), + limits: StoreLimitsBuilder::new() + .memory_size(config.max_memory_bytes) + .table_elements(config.max_table_elements) + .instances(config.max_instances_per_session) + .tables(config.max_tables_per_session) + .memories(config.max_memories_per_session) + .build(), } } } /// Shared wasmtime [`Engine`] configured for the component model with fuel -/// metering enabled, plus the resource limits applied to every guest store -/// created from it. +/// metering and epoch-based interruption enabled, the pooling allocator +/// installed, plus the resource limits applied to every guest store created +/// from it. /// /// One [`WasmDeciderEngine`] is meant to be built once per process and shared -/// across every loaded [`crate::WasmDeciderModule`]. +/// across every loaded [`crate::WasmDeciderModule`]. Building one also starts +/// a background thread that ticks the engine's epoch on a fixed cadence; the +/// thread holds only a weak reference to the underlying wasmtime [`Engine`] +/// and exits on its own once every clone of it has been dropped. #[derive(Clone)] pub struct WasmDeciderEngine { engine: Engine, @@ -79,8 +248,14 @@ pub struct WasmDeciderEngine { /// Failure building the shared wasmtime engine. #[derive(Debug, thiserror::Error)] -#[error("failed to configure wasmtime engine")] -pub struct WasmEngineError(#[source] wasmtime::Error); +pub enum WasmEngineError { + /// Wasmtime rejected the configuration. + #[error("failed to configure wasmtime engine")] + Configure(#[source] wasmtime::Error), + /// The background epoch ticker thread could not be started. + #[error("failed to start the epoch interruption ticker thread")] + EpochTicker(#[source] std::io::Error), +} impl WasmDeciderEngine { /// Builds a new engine with the given resource limits. @@ -88,7 +263,10 @@ impl WasmDeciderEngine { let mut wasmtime_config = wasmtime::Config::new(); wasmtime_config.wasm_component_model(true); wasmtime_config.consume_fuel(true); - let engine = Engine::new(&wasmtime_config).map_err(WasmEngineError)?; + wasmtime_config.epoch_interruption(true); + wasmtime_config.allocation_strategy(InstanceAllocationStrategy::Pooling(pooling_allocation_config(&config))); + let engine = Engine::new(&wasmtime_config).map_err(WasmEngineError::Configure)?; + spawn_epoch_ticker(engine.weak(), config.epoch_tick_interval).map_err(WasmEngineError::EpochTicker)?; Ok(Self { engine, config }) } @@ -102,13 +280,76 @@ impl WasmDeciderEngine { self.config } - /// Creates a fresh per-session store with the memory limiter installed, so - /// wasmtime enforces the configured ceiling on `memory.grow`. + /// Creates a fresh per-session store with the resource limiter installed, + /// so wasmtime enforces the configured ceilings on memory, table, and + /// instance growth. pub(crate) fn new_store(&self) -> Store { let mut store = Store::new(&self.engine, GuestState::new(&self.config)); store.limiter(|state| &mut state.limits); store } + + /// Arms a guest export call with both a fuel budget and a wall-clock + /// epoch deadline. + /// + /// Call this immediately before every guest export call so a + /// fuel-cheap-but-slow guest still traps within a bounded amount of wall + /// clock time, not only a bounded instruction count. + pub(crate) fn arm_guest_call(&self, store: &mut Store, fuel: u64, epoch_ticks: u64) -> wasmtime::Result<()> { + store.set_fuel(fuel)?; + store.set_epoch_deadline(epoch_ticks); + Ok(()) + } +} + +/// Sizes the pooling allocator consistently with the per-session resource +/// limits: the per-component maximums match the per-store [`StoreLimits`] +/// installed by [`GuestState::new`], and the pool-wide totals are that many +/// resources times [`WasmEngineConfig::max_concurrent_sessions`]. +fn pooling_allocation_config(config: &WasmEngineConfig) -> PoolingAllocationConfig { + let sessions = saturating_u32(config.max_concurrent_sessions); + let instances_per_session = saturating_u32(config.max_instances_per_session); + let tables_per_session = saturating_u32(config.max_tables_per_session); + let memories_per_session = saturating_u32(config.max_memories_per_session); + + let mut pooling = PoolingAllocationConfig::new(); + pooling + .total_component_instances(sessions) + .total_core_instances(sessions.saturating_mul(instances_per_session)) + .total_tables(sessions.saturating_mul(tables_per_session)) + .total_memories(sessions.saturating_mul(memories_per_session)) + .max_core_instances_per_component(instances_per_session) + .max_tables_per_component(tables_per_session) + .max_memories_per_component(memories_per_session) + .table_elements(config.max_table_elements) + .max_memory_size(config.max_memory_bytes); + pooling +} + +fn saturating_u32(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + +/// Starts the background thread that periodically increments the wasmtime +/// engine's epoch, driving epoch-based interruption. +/// +/// The thread only holds an [`EngineWeak`], so it never keeps the engine +/// alive on its own; once every [`WasmDeciderEngine`] (and therefore every +/// clone of the underlying [`Engine`]) is dropped, the next tick observes a +/// failed upgrade and the thread exits. +fn spawn_epoch_ticker(engine: EngineWeak, tick_interval: Duration) -> std::io::Result<()> { + std::thread::Builder::new() + .name("wasm-decider-epoch-ticker".to_string()) + .spawn(move || { + loop { + std::thread::sleep(tick_interval); + match engine.upgrade() { + Some(engine) => engine.increment_epoch(), + None => break, + } + } + }) + .map(|_join_handle| ()) } #[cfg(test)] diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine/tests.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine/tests.rs index a338801c7..ddeefd5e8 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine/tests.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/engine/tests.rs @@ -1,3 +1,5 @@ +use wasmtime::{Instance, Memory, MemoryType, Module, Ref, RefType, Table, TableType}; + use super::*; #[test] @@ -5,15 +7,36 @@ fn default_config_uses_documented_defaults() { let config = WasmEngineConfig::default(); assert_eq!(config.max_memory_bytes(), DEFAULT_MAX_MEMORY_BYTES); assert_eq!(config.fuel_per_call(), DEFAULT_FUEL_PER_CALL); + assert_eq!(config.max_table_elements(), DEFAULT_MAX_TABLE_ELEMENTS); + assert_eq!(config.max_instances_per_session(), DEFAULT_MAX_INSTANCES_PER_SESSION); + assert_eq!(config.max_tables_per_session(), DEFAULT_MAX_TABLES_PER_SESSION); + assert_eq!(config.max_memories_per_session(), DEFAULT_MAX_MEMORIES_PER_SESSION); + assert_eq!(config.max_concurrent_sessions(), DEFAULT_MAX_CONCURRENT_SESSIONS); + assert_eq!(config.epoch_tick_interval(), DEFAULT_EPOCH_TICK_INTERVAL); + assert_eq!(config.epoch_ticks_per_call(), DEFAULT_EPOCH_TICKS_PER_CALL); } #[test] fn builder_methods_override_defaults() { let config = WasmEngineConfig::default() .with_max_memory_bytes(4096) - .with_fuel_per_call(1); + .with_fuel_per_call(1) + .with_max_table_elements(16) + .with_max_instances_per_session(2) + .with_max_tables_per_session(3) + .with_max_memories_per_session(4) + .with_max_concurrent_sessions(5) + .with_epoch_tick_interval(Duration::from_millis(7)) + .with_epoch_ticks_per_call(9); assert_eq!(config.max_memory_bytes(), 4096); assert_eq!(config.fuel_per_call(), 1); + assert_eq!(config.max_table_elements(), 16); + assert_eq!(config.max_instances_per_session(), 2); + assert_eq!(config.max_tables_per_session(), 3); + assert_eq!(config.max_memories_per_session(), 4); + assert_eq!(config.max_concurrent_sessions(), 5); + assert_eq!(config.epoch_tick_interval(), Duration::from_millis(7)); + assert_eq!(config.epoch_ticks_per_call(), 9); } #[test] @@ -23,9 +46,90 @@ fn new_builds_a_component_model_engine() { } #[test] -fn engine_error_preserves_the_wasmtime_source() { - let error = WasmEngineError(wasmtime::Error::msg("boom")); +fn engine_error_preserves_the_wasmtime_configure_source() { + let error = WasmEngineError::Configure(wasmtime::Error::msg("boom")); assert_eq!(error.to_string(), "failed to configure wasmtime engine"); let source = std::error::Error::source(&error).expect("source is preserved"); assert_eq!(source.to_string(), "boom"); } + +#[test] +fn engine_error_preserves_the_epoch_ticker_source() { + let error = WasmEngineError::EpochTicker(std::io::Error::other("boom")); + assert_eq!( + error.to_string(), + "failed to start the epoch interruption ticker thread" + ); + let source = std::error::Error::source(&error).expect("source is preserved"); + assert_eq!(source.to_string(), "boom"); +} + +#[test] +fn memory_growth_beyond_the_configured_ceiling_is_rejected() { + let engine = + WasmDeciderEngine::new(WasmEngineConfig::default().with_max_memory_bytes(64 * 1024)).expect("engine builds"); + let mut store = engine.new_store(); + let memory = Memory::new(&mut store, MemoryType::new(1, None)).expect("memory at the ceiling is allowed"); + assert!(memory.grow(&mut store, 1).is_err()); +} + +#[test] +fn table_growth_beyond_the_configured_ceiling_is_rejected() { + let engine = WasmDeciderEngine::new(WasmEngineConfig::default().with_max_table_elements(4)).expect("engine builds"); + let mut store = engine.new_store(); + let table = Table::new(&mut store, TableType::new(RefType::FUNCREF, 4, None), Ref::Func(None)) + .expect("table at the ceiling is allowed"); + assert!(table.grow(&mut store, 1, Ref::Func(None)).is_err()); +} + +/// The minimal valid core wasm module: just the `\0asm` magic and version +/// header, no sections. Used instead of the WAT text format because this +/// crate disables wasmtime's default features and does not pull in the `wat` +/// text-to-binary parser. +const EMPTY_MODULE_BYTES: [u8; 8] = [0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00]; + +#[test] +fn instance_count_beyond_the_configured_ceiling_is_rejected() { + let engine = + WasmDeciderEngine::new(WasmEngineConfig::default().with_max_instances_per_session(1)).expect("engine builds"); + let module = Module::new(engine.engine(), EMPTY_MODULE_BYTES).expect("trivial module compiles"); + let mut store = engine.new_store(); + Instance::new(&mut store, &module, &[]).expect("first instance is allowed"); + assert!(Instance::new(&mut store, &module, &[]).is_err()); +} + +#[test] +fn arm_guest_call_sets_fuel_and_a_non_zero_epoch_deadline() { + let engine = WasmDeciderEngine::new(WasmEngineConfig::default()).expect("engine builds"); + let mut store = engine.new_store(); + assert!(engine.arm_guest_call(&mut store, 100, 10).is_ok()); +} + +/// The minimal core wasm module with one exported function that does nothing. +const EMPTY_EXPORTED_FUNCTION_MODULE_BYTES: [u8; 31] = [ + 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x04, 0x01, 0x60, 0x00, 0x00, 0x03, 0x02, 0x01, 0x00, 0x07, + 0x05, 0x01, 0x01, 0x66, 0x00, 0x00, 0x0a, 0x04, 0x01, 0x02, 0x00, 0x0b, +]; + +#[test] +fn a_zero_tick_epoch_deadline_traps_a_guest_call() { + let engine = WasmDeciderEngine::new(WasmEngineConfig::default()).expect("engine builds"); + let module = Module::new(engine.engine(), EMPTY_EXPORTED_FUNCTION_MODULE_BYTES).expect("trivial module compiles"); + let mut store = engine.new_store(); + let instance = Instance::new(&mut store, &module, &[]).expect("instance"); + let func = instance + .get_typed_func::<(), ()>(&mut store, "f") + .expect("exported function f"); + + engine + .arm_guest_call(&mut store, 1_000_000, 0) + .expect("arming a zero-tick deadline succeeds"); + let error = func + .call(&mut store, ()) + .expect_err("a zero-tick deadline is already elapsed"); + + assert!( + matches!(error.downcast_ref::(), Some(wasmtime::Trap::Interrupt)), + "{error}" + ); +} diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution.rs index facaf294f..16f90c6b4 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution.rs @@ -19,18 +19,124 @@ //! own events. This execution evolves the session with the decided events //! before calling `snapshot`, so a session resumed from that snapshot ends up //! in the same state as a full replay from the beginning of the stream. +//! +//! # Guest calls run off the async executor +//! +//! Every guest export call is fuel-metered and wall-clock bounded (see +//! [`crate::WasmDeciderEngine::arm_guest_call`]), but a synchronous guest call +//! still occupies whatever thread calls it for up to that budget. Running it +//! inline on the async executor would stall every other task that executor +//! thread is responsible for. Both [`WithoutSnapshotStore`] and +//! [`WithSnapshotStore`] executions instead move each contiguous run of guest +//! calls onto a blocking thread pool via [`spawn_guest`], resuming the async +//! task only for real I/O (`StreamRead`, `StreamAppend`, `SnapshotRead`). + +use std::sync::OnceLock; +use std::time::{Duration, Instant}; +use opentelemetry::metrics::{Counter, Histogram}; +use opentelemetry::{KeyValue, global}; use thiserror::Error; use trogon_decider_runtime::{ - AppendStreamRequest, Event, EventId, Headers, ReadAfterOverflow, ReadFrom, ReadSnapshotRequest, ReadStreamRequest, - Snapshot, SnapshotAheadOfStream, SnapshotRead, SnapshotTaskScheduler, SnapshotWrite, StreamAppend, StreamPosition, - StreamRead, StreamWritePrecondition, WriteSnapshotRequest, + AppendStreamRequest, DiscardAndReplaySnapshotFailure, Event, EventId, FailOnSnapshotFailure, Headers, + ReadAfterOverflow, ReadFrom, ReadSnapshotRequest, ReadStreamRequest, Snapshot, SnapshotAheadOfStream, + SnapshotFailure, SnapshotFailureDecision, SnapshotRead, SnapshotTaskScheduler, SnapshotWrite, StreamAppend, + StreamPosition, StreamRead, StreamWritePrecondition, WriteSnapshotRequest, }; use trogon_decider_wit::host::{self, AnyEnvelope, CommandEnvelope, DecideError}; +use trogon_semconv::{attribute, metric, span}; use trogon_std::NowV7; use wasmtime::Store; -use crate::{DomainErrorDetail, OpaqueSnapshotPayload, WasmDeciderModule, WasmSnapshotId}; +use crate::{DomainErrorDetail, OpaqueSnapshotPayload, WasmDeciderEngine, WasmDeciderModule, WasmSnapshotId}; + +const METER_NAME: &str = "trogon-decider-wasm-runtime"; + +struct WasmExecutionMetrics { + execution_duration: Histogram, + fuel_consumed: Histogram, + traps: Counter, +} + +impl WasmExecutionMetrics { + fn new() -> Self { + let meter = global::meter(METER_NAME); + Self { + execution_duration: metric::build_decider_wasm_execution_duration(&meter), + fuel_consumed: metric::build_decider_wasm_fuel_consumed(&meter), + traps: metric::build_decider_wasm_traps(&meter), + } + } +} + +static METRICS: OnceLock = OnceLock::new(); + +fn metrics() -> &'static WasmExecutionMetrics { + METRICS.get_or_init(WasmExecutionMetrics::new) +} + +/// Module, version, and command type shared by every guest phase span and +/// metric recorded for one command execution. +#[derive(Debug, Clone)] +struct GuestPhaseContext { + module_name: String, + module_version: String, + command_type: String, +} + +impl GuestPhaseContext { + fn new(module: &WasmDeciderModule, command: &CommandEnvelope) -> Self { + Self { + module_name: module.name().to_string(), + module_version: module.version().to_string(), + command_type: command.type_.clone(), + } + } +} + +fn phase_attributes(context: &GuestPhaseContext, phase: attribute::GuestPhase) -> [KeyValue; 4] { + [ + KeyValue::new(attribute::MODULE_NAME, context.module_name.clone()), + KeyValue::new(attribute::MODULE_VERSION, context.module_version.clone()), + KeyValue::new(attribute::COMMAND_TYPE, context.command_type.clone()), + KeyValue::new(attribute::GUEST_PHASE, phase.as_str()), + ] +} + +fn record_phase_metrics( + context: &GuestPhaseContext, + phase: attribute::GuestPhase, + duration: Duration, + fuel_consumed: u64, +) { + let attributes = phase_attributes(context, phase); + metrics().execution_duration.record(duration.as_secs_f64(), &attributes); + metrics().fuel_consumed.record(fuel_consumed, &attributes); +} + +fn record_phase_trap( + context: &GuestPhaseContext, + phase: attribute::GuestPhase, + classification: attribute::TrapClassification, +) { + let mut attributes = phase_attributes(context, phase).to_vec(); + attributes.push(KeyValue::new(attribute::TRAP_CLASSIFICATION, classification.as_str())); + metrics().traps.add(1, &attributes); +} + +fn phase_fuel_consumed(store: &Store, fuel_budget: u64) -> u64 { + store + .get_fuel() + .map_or(fuel_budget, |remaining| fuel_budget.saturating_sub(remaining)) +} + +fn trap_classification(error: &wasmtime::Error) -> attribute::TrapClassification { + if is_epoch_deadline_exceeded(error) { + attribute::TrapClassification::DeadlineExceeded + } else { + attribute::TrapClassification::Trap + } +} /// Result of a successful WASM command execution. #[derive(Debug, Clone)] @@ -59,9 +165,18 @@ pub enum WasmCommandError /// A guest call trapped (fuel exhaustion, memory limit, or ABI failure). /// /// Distinct from [`Self::Faulted`]: a trap is a host-level wasmtime - /// failure, not a domain error the guest chose to report. + /// failure, not a domain error the guest chose to report. Distinct from + /// [`Self::DeadlineExceeded`]: this variant is every other trap cause. #[error("guest call trapped")] Trap(#[source] wasmtime::Error), + /// A guest call exceeded its wall-clock epoch deadline. + /// + /// Wasmtime's epoch-based interruption raised this instead of the guest + /// exhausting its fuel budget first, meaning the guest was still running + /// but too slowly. Distinguished from [`Self::Trap`] so callers can tell + /// a hung guest apart from every other host-level trap. + #[error("guest call exceeded its wall-clock deadline")] + DeadlineExceeded(#[source] wasmtime::Error), /// The module could not be instantiated for this command. #[error("failed to instantiate wasm component")] Instantiate(#[source] wasmtime::Error), @@ -80,16 +195,77 @@ pub enum WasmCommandError /// The snapshot's recorded position cannot be advanced (u64 overflow). #[error("{0}")] ReadAfterOverflow(#[source] ReadAfterOverflow), + /// The blocking task running guest calls panicked or was cancelled. + #[error("guest execution task failed")] + Blocking(#[source] tokio::task::JoinError), +} + +/// Module identity, command type, and stream id a [`WasmSnapshotFailurePolicy`] +/// inspects before deciding how to react to a snapshot a [`WasmCommandExecution`] +/// cannot trust. +/// +/// Mirrors [`trogon_decider_runtime::SnapshotFailureContext`], adapted for the +/// WASM boundary: there is no typed `Decider` to hand the policy, so this +/// carries the identity the wasm execution actually has in hand at this point +/// instead of a command reference. +#[derive(Debug)] +pub struct WasmSnapshotFailureContext<'a, ReadSnapshotError> { + /// Name of the module executing the command that triggered this failure. + pub module_name: &'a str, + /// Version of the module executing the command that triggered this failure. + pub module_version: &'a str, + /// Wire type URL of the command that triggered this failure. + pub command_type: &'a str, + /// Stream id the command resolved before loading its snapshot. + pub stream_id: &'a str, + /// The failure the policy must decide how to handle. + pub failure: SnapshotFailure<'a, ReadSnapshotError>, +} + +/// Chooses how a [`WasmCommandExecution`] reacts to a snapshot it cannot trust. +/// +/// Mirrors [`trogon_decider_runtime::SnapshotFailurePolicy`] for the WASM +/// boundary, which has no typed `Decider` to bound the policy on. +/// [`FailOnSnapshotFailure`] keeps today's behavior of failing the command; +/// [`DiscardAndReplaySnapshotFailure`] discards the bad snapshot and replays +/// from the beginning of the stream. Both are reused directly from +/// `trogon_decider_runtime` since neither carries decider-specific state. +pub trait WasmSnapshotFailurePolicy { + /// Decides how the command execution should react to the given snapshot failure. + fn decide_snapshot_failure( + &self, + context: WasmSnapshotFailureContext<'_, ReadSnapshotError>, + ) -> SnapshotFailureDecision; +} + +impl WasmSnapshotFailurePolicy for FailOnSnapshotFailure { + fn decide_snapshot_failure( + &self, + _context: WasmSnapshotFailureContext<'_, ReadSnapshotError>, + ) -> SnapshotFailureDecision { + SnapshotFailureDecision::Fail + } +} + +impl WasmSnapshotFailurePolicy for DiscardAndReplaySnapshotFailure { + fn decide_snapshot_failure( + &self, + _context: WasmSnapshotFailureContext<'_, ReadSnapshotError>, + ) -> SnapshotFailureDecision { + SnapshotFailureDecision::DiscardAndReplay + } } /// Marker type used before a snapshot store is attached to an execution. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct WithoutSnapshotStore; -/// Snapshot store and best-effort task scheduler attached to an execution. -pub struct WithSnapshotStore<'a, S, Sched> { +/// Snapshot store, best-effort task scheduler, and snapshot failure policy +/// attached to an execution. +pub struct WithSnapshotStore<'a, S, Sched, F = FailOnSnapshotFailure> { store: &'a S, task_scheduler: &'a Sched, + failure_policy: F, } /// Builder for one command execution against a [`WasmDeciderModule`]. @@ -120,11 +296,16 @@ impl<'a, E> WasmCommandExecution<'a, E, WithoutSnapshotStore, trogon_std::UuidV7 impl<'a, E, G> WasmCommandExecution<'a, E, WithoutSnapshotStore, G> { /// Attaches a snapshot store and best-effort snapshot task scheduler. + /// + /// Defaults to [`FailOnSnapshotFailure`], which fails the command on an + /// untrusted snapshot exactly as before this policy existed. Chain + /// [`WasmCommandExecution::with_snapshot_failure_policy`] to recover + /// instead. pub fn with_snapshot_store( self, snapshot_store: &'a S, snapshot_task_scheduler: &'a Sched, - ) -> WasmCommandExecution<'a, E, WithSnapshotStore<'a, S, Sched>, G> { + ) -> WasmCommandExecution<'a, E, WithSnapshotStore<'a, S, Sched, FailOnSnapshotFailure>, G> { WasmCommandExecution { module: self.module, event_store: self.event_store, @@ -132,6 +313,33 @@ impl<'a, E, G> WasmCommandExecution<'a, E, WithoutSnapshotStore, G> { snapshots: WithSnapshotStore { store: snapshot_store, task_scheduler: snapshot_task_scheduler, + failure_policy: FailOnSnapshotFailure, + }, + write_precondition: self.write_precondition, + headers: self.headers, + event_id_generator: self.event_id_generator, + } + } +} + +impl<'a, E, S, Sched, F, G> WasmCommandExecution<'a, E, WithSnapshotStore<'a, S, Sched, F>, G> { + /// Sets how this execution reacts to a snapshot it cannot trust. + /// + /// Defaults to [`FailOnSnapshotFailure`], which fails the command exactly + /// as before this policy existed. Use [`DiscardAndReplaySnapshotFailure`] + /// or a custom [`WasmSnapshotFailurePolicy`] to recover instead. + pub fn with_snapshot_failure_policy( + self, + failure_policy: NextF, + ) -> WasmCommandExecution<'a, E, WithSnapshotStore<'a, S, Sched, NextF>, G> { + WasmCommandExecution { + module: self.module, + event_store: self.event_store, + command: self.command, + snapshots: WithSnapshotStore { + store: self.snapshots.store, + task_scheduler: self.snapshots.task_scheduler, + failure_policy, }, write_precondition: self.write_precondition, headers: self.headers, @@ -178,14 +386,13 @@ impl<'a, E, Snapshots, G> WasmCommandExecution<'a, E, Snapshots, G> { } /// Prior stream events replayed as guest envelopes, the stream's current -/// position, the snapshot position they resumed from (if any), and the raw -/// snapshot bytes passed to the guest session constructor. -type ReplayContext = ( - Vec, - Option, - Option, - Option>, -); +/// position, and the raw snapshot bytes passed to the guest session +/// constructor. +struct ReplayContext { + replayed_envelopes: Vec, + current_position: Option, + snapshot_bytes: Option>, +} impl WasmCommandExecution<'_, E, WithoutSnapshotStore, G> where @@ -199,12 +406,19 @@ where WasmExecutionResult, WasmCommandError>::Error, >::Error>, > { - let engine = self.module.engine(); - let mut store = engine.new_store(); - let bindings = instantiate(&mut store, self.module)?; - let stream_id = call_stream_id(&mut store, &bindings, engine, self.command)?; - let write_precondition = command_write_precondition(self.module, &self.command.type_); + let engine = self.module.engine().clone(); + let decider_pre = self.module.decider_pre().clone(); + let command = self.command.clone(); + let phase_context = GuestPhaseContext::new(self.module, self.command); + let (mut store, bindings, stream_id) = spawn_guest(move || { + let mut store = engine.new_store(); + let bindings = instantiate(&mut store, &decider_pre, &engine, &phase_context)?; + let stream_id = call_stream_id(&mut store, &bindings, &engine, &command)?; + Ok((store, bindings, stream_id)) + }) + .await?; + let write_precondition = command_write_precondition(self.module, &self.command.type_); let (replayed_envelopes, current_position) = if is_no_stream(write_precondition) { (Vec::new(), None) } else { @@ -221,12 +435,27 @@ where (to_any_envelopes(stream_read.events), current_position) }; - let session = create_session(&mut store, &bindings, engine, None)?; - replay_events(&mut store, &bindings, engine, session, &replayed_envelopes)?; - let decided_envelopes = decide(&mut store, &bindings, engine, session, self.command)?; - // No snapshot observes this session, so the decided events are not - // folded back into it; folding here would only burn guest fuel. - host::drop_session(&bindings, &mut store, session).map_err(WasmCommandError::Trap)?; + let engine = self.module.engine().clone(); + let command = self.command.clone(); + let phase_context = GuestPhaseContext::new(self.module, self.command); + let decided_envelopes = spawn_guest(move || { + let session = create_session(&mut store, &bindings, &engine, None, &phase_context)?; + replay_events( + &mut store, + &bindings, + &engine, + session, + &replayed_envelopes, + &phase_context, + )?; + let decided_envelopes = decide(&mut store, &bindings, &engine, session, &command, &phase_context)?; + // No snapshot observes this session, so the decided events are not + // folded back into it; folding here would only burn guest fuel. + host::drop_session(&bindings, &mut store, session) + .map_err(|error| map_trap(error, WasmCommandError::Trap))?; + Ok(decided_envelopes) + }) + .await?; let precondition = resolve_write_precondition(write_precondition, self.write_precondition, current_position); let events = encode_events(decided_envelopes.clone(), &self.headers, &self.event_id_generator); @@ -248,7 +477,7 @@ where } } -impl WasmCommandExecution<'_, E, WithSnapshotStore<'_, S, Sched>, G> +impl WasmCommandExecution<'_, E, WithSnapshotStore<'_, S, Sched, F>, G> where E: StreamRead + StreamAppend, S: SnapshotRead @@ -258,6 +487,7 @@ where + Sync + 'static, Sched: SnapshotTaskScheduler, + F: WasmSnapshotFailurePolicy<>::Error>, G: NowV7, { /// Runs the command against a fresh guest session, appends its decided @@ -274,29 +504,67 @@ where >::Error, >, > { - let engine = self.module.engine(); - let mut store = engine.new_store(); - let bindings = instantiate(&mut store, self.module)?; - let stream_id = call_stream_id(&mut store, &bindings, engine, self.command)?; - let write_precondition = command_write_precondition(self.module, &self.command.type_); - - let (replayed_envelopes, current_position, _snapshot_position, snapshot_bytes) = - if is_no_stream(write_precondition) { - (Vec::new(), None, None, None) - } else { - self.load_replay_context(&stream_id).await? - }; + let engine = self.module.engine().clone(); + let decider_pre = self.module.decider_pre().clone(); + let command = self.command.clone(); + let phase_context = GuestPhaseContext::new(self.module, self.command); + let (mut store, bindings, stream_id) = spawn_guest(move || { + let mut store = engine.new_store(); + let bindings = instantiate(&mut store, &decider_pre, &engine, &phase_context)?; + let stream_id = call_stream_id(&mut store, &bindings, &engine, &command)?; + Ok((store, bindings, stream_id)) + }) + .await?; - let session = create_session(&mut store, &bindings, engine, snapshot_bytes.as_deref())?; - replay_events(&mut store, &bindings, engine, session, &replayed_envelopes)?; - let decided_envelopes = decide(&mut store, &bindings, engine, session, self.command)?; - fold_decided_events(&mut store, &bindings, engine, session, &decided_envelopes)?; + let write_precondition = command_write_precondition(self.module, &self.command.type_); + let ReplayContext { + replayed_envelopes, + current_position, + snapshot_bytes, + } = if is_no_stream(write_precondition) { + ReplayContext { + replayed_envelopes: Vec::new(), + current_position: None, + snapshot_bytes: None, + } + } else { + self.load_replay_context(&stream_id).await? + }; - store - .set_fuel(engine.config().fuel_per_call()) - .map_err(WasmCommandError::Trap)?; - let new_snapshot_bytes = host::snapshot(&bindings, &mut store, session).map_err(WasmCommandError::Trap)?; - host::drop_session(&bindings, &mut store, session).map_err(WasmCommandError::Trap)?; + let engine = self.module.engine().clone(); + let command = self.command.clone(); + let phase_context = GuestPhaseContext::new(self.module, self.command); + let (decided_envelopes, new_snapshot_bytes) = spawn_guest(move || { + let session = create_session( + &mut store, + &bindings, + &engine, + snapshot_bytes.as_deref(), + &phase_context, + )?; + replay_events( + &mut store, + &bindings, + &engine, + session, + &replayed_envelopes, + &phase_context, + )?; + let decided_envelopes = decide(&mut store, &bindings, &engine, session, &command, &phase_context)?; + fold_decided_events( + &mut store, + &bindings, + &engine, + session, + &decided_envelopes, + &phase_context, + )?; + let new_snapshot_bytes = take_snapshot(&mut store, &bindings, &engine, session, &phase_context)?; + host::drop_session(&bindings, &mut store, session) + .map_err(|error| map_trap(error, WasmCommandError::Trap))?; + Ok((decided_envelopes, new_snapshot_bytes)) + }) + .await?; let precondition = resolve_write_precondition(write_precondition, self.write_precondition, current_position); let events = encode_events(decided_envelopes.clone(), &self.headers, &self.event_id_generator); @@ -327,6 +595,32 @@ where }) } + /// Builds the context a [`WasmSnapshotFailurePolicy`] inspects for one + /// snapshot failure this execution encountered. + fn snapshot_failure_context<'ctx, ReadSnapshotError>( + &'ctx self, + stream_id: &'ctx str, + failure: SnapshotFailure<'ctx, ReadSnapshotError>, + ) -> WasmSnapshotFailureContext<'ctx, ReadSnapshotError> { + WasmSnapshotFailureContext { + module_name: self.module.name().as_str(), + module_version: self.module.version().as_str(), + command_type: self.command.type_.as_str(), + stream_id, + failure, + } + } + + /// Loads the snapshot (if any) and the stream events replayed after it. + /// + /// A snapshot the configured [`WasmSnapshotFailurePolicy`] cannot trust, + /// whether it failed to read or claims a position ahead of the stream, is + /// routed through that policy. [`SnapshotFailureDecision::Fail`] returns + /// the concrete [`WasmCommandError`] for the failure, matching this + /// execution's behavior before the policy existed. + /// [`SnapshotFailureDecision::DiscardAndReplay`] discards the untrusted + /// snapshot and replays the stream from the beginning instead, exactly as + /// [`trogon_decider_runtime::CommandExecution`] does natively. #[allow(clippy::type_complexity)] async fn load_replay_context( &self, @@ -340,18 +634,27 @@ where >, > { let snapshot_id = WasmSnapshotId::new(self.module.name(), self.module.version(), stream_id); - let response = >::read_snapshot( - self.snapshots.store, - ReadSnapshotRequest { - snapshot_id: snapshot_id.as_str(), - }, - ) - .await - .map_err(WasmCommandError::ReadSnapshot)?; - let (snapshot_position, snapshot_bytes) = match response.snapshot { - Some(snapshot) => (Some(snapshot.position), Some(snapshot.payload.into_bytes())), - None => (None, None), - }; + let (snapshot_position, mut snapshot_bytes) = + match >::read_snapshot( + self.snapshots.store, + ReadSnapshotRequest { + snapshot_id: snapshot_id.as_str(), + }, + ) + .await + { + Ok(response) => match response.snapshot { + Some(snapshot) => (Some(snapshot.position), Some(snapshot.payload.into_bytes())), + None => (None, None), + }, + Err(error) => { + let context = self.snapshot_failure_context(stream_id, SnapshotFailure::ReadFailed(&error)); + match self.snapshots.failure_policy.decide_snapshot_failure(context) { + SnapshotFailureDecision::Fail => return Err(WasmCommandError::ReadSnapshot(error)), + SnapshotFailureDecision::DiscardAndReplay => (None, None), + } + } + }; let from = match snapshot_position { Some(position) => ReadFrom::after(position).map_err(WasmCommandError::ReadAfterOverflow)?, @@ -360,77 +663,209 @@ where let stream_read = >::read_stream(self.event_store, ReadStreamRequest { stream_id, from }) .await .map_err(WasmCommandError::ReadStream)?; - let current_position = stream_read.current_position; + let mut current_position = stream_read.current_position; + let mut replayed_envelopes = to_any_envelopes(stream_read.events); + + if let Some(position) = snapshot_position + && let Err(ahead_of_stream) = ensure_snapshot_not_ahead(position, current_position) + { + let context = self.snapshot_failure_context(stream_id, SnapshotFailure::AheadOfStream(ahead_of_stream)); + match self.snapshots.failure_policy.decide_snapshot_failure(context) { + SnapshotFailureDecision::Fail => { + return Err(WasmCommandError::SnapshotAheadOfStream(ahead_of_stream)); + } + SnapshotFailureDecision::DiscardAndReplay => { + snapshot_bytes = None; - if let Some(snapshot_position) = snapshot_position { - ensure_snapshot_not_ahead(snapshot_position, current_position) - .map_err(WasmCommandError::SnapshotAheadOfStream)?; + let replay = >::read_stream( + self.event_store, + ReadStreamRequest { + stream_id, + from: ReadFrom::Beginning, + }, + ) + .await + .map_err(WasmCommandError::ReadStream)?; + current_position = replay.current_position; + replayed_envelopes = to_any_envelopes(replay.events); + } + } } - Ok(( - to_any_envelopes(stream_read.events), + Ok(ReplayContext { + replayed_envelopes, current_position, - snapshot_position, snapshot_bytes, - )) + }) + } +} + +/// Runs a synchronous, guest-touching closure on a blocking thread pool so it +/// never occupies the async executor for the duration of its fuel and epoch +/// budget. See the module-level doc comment for why this is necessary. +async fn spawn_guest( + task: impl FnOnce() -> Result> + + Send + + 'static, +) -> Result> +where + T: Send + 'static, + ReadSnapshotError: Send + 'static, + ReadStreamError: Send + 'static, + AppendStreamError: Send + 'static, +{ + match tokio::task::spawn_blocking(task).await { + Ok(result) => result, + Err(join_error) => Err(WasmCommandError::Blocking(join_error)), + } +} + +fn is_epoch_deadline_exceeded(error: &wasmtime::Error) -> bool { + matches!(error.downcast_ref::(), Some(wasmtime::Trap::Interrupt)) +} + +/// Classifies a wasmtime call failure, surfacing an epoch-deadline interrupt +/// as [`WasmCommandError::DeadlineExceeded`] and delegating every other +/// failure to `fallback`. +fn map_trap( + error: wasmtime::Error, + fallback: impl FnOnce(wasmtime::Error) -> WasmCommandError, +) -> WasmCommandError { + if is_epoch_deadline_exceeded(&error) { + WasmCommandError::DeadlineExceeded(error) + } else { + fallback(error) } } fn instantiate( store: &mut Store, - module: &WasmDeciderModule, + decider_pre: &host::DeciderPre, + engine: &WasmDeciderEngine, + context: &GuestPhaseContext, ) -> Result> { - store - .set_fuel(module.engine().config().fuel_per_call()) - .map_err(WasmCommandError::Trap)?; - module - .decider_pre() - .instantiate(store) - .map_err(WasmCommandError::Instantiate) + let span = tracing::info_span!( + span::DECIDER_WASM_INSTANTIATE, + module_name = %context.module_name, + module_version = %context.module_version, + command_type = %context.command_type, + guest_phase = attribute::GuestPhase::Instantiate.as_str(), + trap_classification = tracing::field::Empty, + ); + span.in_scope(|| { + let fuel_budget = engine.config().fuel_per_call(); + engine + .arm_guest_call(store, fuel_budget, engine.config().epoch_ticks_per_call()) + .map_err(WasmCommandError::Trap)?; + let start = Instant::now(); + let result = decider_pre.instantiate(&mut *store); + let duration = start.elapsed(); + let fuel_consumed = phase_fuel_consumed(store, fuel_budget); + record_phase_metrics(context, attribute::GuestPhase::Instantiate, duration, fuel_consumed); + result.map_err(|error| { + let classification = trap_classification(&error); + span.record(attribute::TRAP_CLASSIFICATION, classification.as_str()); + record_phase_trap(context, attribute::GuestPhase::Instantiate, classification); + map_trap(error, WasmCommandError::Instantiate) + }) + }) } fn call_stream_id( store: &mut Store, bindings: &host::Decider, - engine: &crate::WasmDeciderEngine, + engine: &WasmDeciderEngine, command: &CommandEnvelope, ) -> Result> { - store - .set_fuel(engine.config().fuel_per_call()) + engine + .arm_guest_call( + store, + engine.config().fuel_per_call(), + engine.config().epoch_ticks_per_call(), + ) .map_err(WasmCommandError::Trap)?; host::call_stream_id(bindings, store, command) - .map_err(WasmCommandError::Trap)? + .map_err(|error| map_trap(error, WasmCommandError::Trap))? .map_err(|error| WasmCommandError::StreamId(error.into())) } fn create_session( store: &mut Store, bindings: &host::Decider, - engine: &crate::WasmDeciderEngine, + engine: &WasmDeciderEngine, snapshot: Option<&[u8]>, + context: &GuestPhaseContext, ) -> Result> { - store - .set_fuel(engine.config().fuel_per_call()) - .map_err(WasmCommandError::Trap)?; - host::create_session(bindings, store, snapshot).map_err(WasmCommandError::Trap) + let span = tracing::info_span!( + span::DECIDER_WASM_INSTANTIATE, + module_name = %context.module_name, + module_version = %context.module_version, + command_type = %context.command_type, + guest_phase = attribute::GuestPhase::Instantiate.as_str(), + trap_classification = tracing::field::Empty, + ); + span.in_scope(|| { + let fuel_budget = engine.config().fuel_per_call(); + engine + .arm_guest_call(store, fuel_budget, engine.config().epoch_ticks_per_call()) + .map_err(WasmCommandError::Trap)?; + let start = Instant::now(); + let result = host::create_session(bindings, store, snapshot); + let duration = start.elapsed(); + let fuel_consumed = phase_fuel_consumed(store, fuel_budget); + record_phase_metrics(context, attribute::GuestPhase::Instantiate, duration, fuel_consumed); + result.map_err(|error| { + let classification = trap_classification(&error); + span.record(attribute::TRAP_CLASSIFICATION, classification.as_str()); + record_phase_trap(context, attribute::GuestPhase::Instantiate, classification); + map_trap(error, WasmCommandError::Trap) + }) + }) } fn replay_events( store: &mut Store, bindings: &host::Decider, - engine: &crate::WasmDeciderEngine, + engine: &WasmDeciderEngine, session: host::Session, events: &[AnyEnvelope], + context: &GuestPhaseContext, ) -> Result<(), WasmCommandError> { if events.is_empty() { return Ok(()); } - store - .set_fuel(replay_fuel(engine.config().fuel_per_call(), events.len())) - .map_err(WasmCommandError::Trap)?; - host::evolve(bindings, store, session, events) - .map_err(WasmCommandError::Trap)? - .map_err(|error| WasmCommandError::Evolve(error.into())) + let span = tracing::info_span!( + span::DECIDER_WASM_REPLAY, + module_name = %context.module_name, + module_version = %context.module_version, + command_type = %context.command_type, + guest_phase = attribute::GuestPhase::Replay.as_str(), + trap_classification = tracing::field::Empty, + ); + span.in_scope(|| { + let fuel_budget = replay_fuel(engine.config().fuel_per_call(), events.len()); + engine + .arm_guest_call( + store, + fuel_budget, + replay_epoch_ticks(engine.config().epoch_ticks_per_call(), events.len()), + ) + .map_err(WasmCommandError::Trap)?; + let start = Instant::now(); + let result = host::evolve(bindings, store, session, events); + let duration = start.elapsed(); + let fuel_consumed = phase_fuel_consumed(store, fuel_budget); + record_phase_metrics(context, attribute::GuestPhase::Replay, duration, fuel_consumed); + match result { + Ok(inner) => inner.map_err(|error| WasmCommandError::Evolve(error.into())), + Err(error) => { + let classification = trap_classification(&error); + span.record(attribute::TRAP_CLASSIFICATION, classification.as_str()); + record_phase_trap(context, attribute::GuestPhase::Replay, classification); + Err(map_trap(error, WasmCommandError::Trap)) + } + } + }) } /// Fuel budget for one batched replay `evolve` call. @@ -445,19 +880,60 @@ fn replay_fuel(fuel_per_call: u64, event_count: usize) -> u64 { .unwrap_or(u64::MAX) } +/// Epoch tick budget for one batched replay `evolve` call, scaled the same +/// way as [`replay_fuel`] so a long replay gets a proportionally larger +/// wall-clock allowance instead of tripping the single-call deadline. +fn replay_epoch_ticks(epoch_ticks_per_call: u64, event_count: usize) -> u64 { + u64::try_from(event_count) + .map(|count| epoch_ticks_per_call.saturating_mul(count)) + .unwrap_or(u64::MAX) +} + fn decide( store: &mut Store, bindings: &host::Decider, - engine: &crate::WasmDeciderEngine, + engine: &WasmDeciderEngine, session: host::Session, command: &CommandEnvelope, + context: &GuestPhaseContext, ) -> Result, WasmCommandError> { - store - .set_fuel(engine.config().fuel_per_call()) - .map_err(WasmCommandError::Trap)?; - host::decide(bindings, store, session, command) - .map_err(WasmCommandError::Trap)? - .map_err(map_decide_error) + let span = tracing::info_span!( + span::DECIDER_WASM_DECIDE, + module_name = %context.module_name, + module_version = %context.module_version, + command_type = %context.command_type, + guest_phase = attribute::GuestPhase::Decide.as_str(), + decision_outcome = tracing::field::Empty, + trap_classification = tracing::field::Empty, + ); + span.in_scope(|| { + let fuel_budget = engine.config().fuel_per_call(); + engine + .arm_guest_call(store, fuel_budget, engine.config().epoch_ticks_per_call()) + .map_err(WasmCommandError::Trap)?; + let start = Instant::now(); + let result = host::decide(bindings, store, session, command); + let duration = start.elapsed(); + let fuel_consumed = phase_fuel_consumed(store, fuel_budget); + record_phase_metrics(context, attribute::GuestPhase::Decide, duration, fuel_consumed); + match result { + Ok(inner) => { + let decision_outcome = match &inner { + Ok(_) => attribute::DecisionOutcome::Decided, + Err(DecideError::Rejected(_)) => attribute::DecisionOutcome::Rejected, + Err(DecideError::Faulted(_)) => attribute::DecisionOutcome::Faulted, + }; + span.record(attribute::DECISION_OUTCOME, decision_outcome.as_str()); + inner.map_err(map_decide_error) + } + Err(error) => { + let classification = trap_classification(&error); + span.record(attribute::TRAP_CLASSIFICATION, classification.as_str()); + record_phase_trap(context, attribute::GuestPhase::Decide, classification); + Err(map_trap(error, WasmCommandError::Trap)) + } + } + }) } fn map_decide_error( @@ -475,11 +951,50 @@ fn map_decide_error( fn fold_decided_events( store: &mut Store, bindings: &host::Decider, - engine: &crate::WasmDeciderEngine, + engine: &WasmDeciderEngine, session: host::Session, decided_envelopes: &[AnyEnvelope], + context: &GuestPhaseContext, ) -> Result<(), WasmCommandError> { - replay_events(store, bindings, engine, session, decided_envelopes) + replay_events(store, bindings, engine, session, decided_envelopes, context) +} + +/// Captures a snapshot of the guest session's current state. +/// +/// Must run after [`fold_decided_events`] folds this command's own decided +/// events back into the session; see the module-level doc comment. +fn take_snapshot( + store: &mut Store, + bindings: &host::Decider, + engine: &WasmDeciderEngine, + session: host::Session, + context: &GuestPhaseContext, +) -> Result>, WasmCommandError> { + let span = tracing::info_span!( + span::DECIDER_WASM_SNAPSHOT, + module_name = %context.module_name, + module_version = %context.module_version, + command_type = %context.command_type, + guest_phase = attribute::GuestPhase::Snapshot.as_str(), + trap_classification = tracing::field::Empty, + ); + span.in_scope(|| { + let fuel_budget = engine.config().fuel_per_call(); + engine + .arm_guest_call(store, fuel_budget, engine.config().epoch_ticks_per_call()) + .map_err(WasmCommandError::Trap)?; + let start = Instant::now(); + let result = host::snapshot(bindings, store, session); + let duration = start.elapsed(); + let fuel_consumed = phase_fuel_consumed(store, fuel_budget); + record_phase_metrics(context, attribute::GuestPhase::Snapshot, duration, fuel_consumed); + result.map_err(|error| { + let classification = trap_classification(&error); + span.record(attribute::TRAP_CLASSIFICATION, classification.as_str()); + record_phase_trap(context, attribute::GuestPhase::Snapshot, classification); + map_trap(error, WasmCommandError::Trap) + }) + }) } fn to_any_envelopes(stream_events: Vec) -> Vec { diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution/tests.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution/tests.rs index 07eafe95b..d54c6afa4 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution/tests.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/execution/tests.rs @@ -2,6 +2,8 @@ use trogon_std::NowV7; use uuid::Uuid; use super::*; +use crate::WasmEngineConfig; +use crate::test_fixture::schedules_bytes; fn position(value: u64) -> StreamPosition { StreamPosition::try_new(value).expect("test stream position must be non-zero") @@ -115,12 +117,72 @@ fn replay_fuel_saturates_instead_of_overflowing() { assert_eq!(replay_fuel(u64::MAX, usize::MAX), u64::MAX); } +#[test] +fn replay_epoch_ticks_scales_linearly_with_event_count() { + assert_eq!(replay_epoch_ticks(10, 1), 10); + assert_eq!(replay_epoch_ticks(10, 250), 2_500); +} + +#[test] +fn replay_epoch_ticks_saturates_instead_of_overflowing() { + assert_eq!(replay_epoch_ticks(u64::MAX, 2), u64::MAX); + assert_eq!(replay_epoch_ticks(u64::MAX, usize::MAX), u64::MAX); +} + +#[test] +fn a_zero_tick_epoch_deadline_traps_as_deadline_exceeded() { + let engine = WasmDeciderEngine::new(WasmEngineConfig::default()).expect("engine builds"); + let module = WasmDeciderModule::load(engine, &schedules_bytes()).expect("module loads"); + + let engine = module.engine(); + let mut store = engine.new_store(); + let command = CommandEnvelope { + type_: "test.v1.DoesNotMatter".to_string(), + payload: Vec::new(), + }; + let context = GuestPhaseContext::new(&module, &command); + let bindings = instantiate::( + &mut store, + module.decider_pre(), + engine, + &context, + ) + .expect("instantiate succeeds"); + + // Arm the deadline directly and call the raw wit binding rather than the + // `call_stream_id` wrapper above: the wrapper re-arms the deadline from + // `engine.config().epoch_ticks_per_call()` immediately before its guest + // call, which would silently clobber the zero-tick deadline this test + // needs to force a trap. + engine + .arm_guest_call(&mut store, engine.config().fuel_per_call(), 0) + .expect("arming a zero-tick deadline succeeds"); + let wasmtime_error = + host::call_stream_id(&bindings, &mut store, &command).expect_err("a zero-tick deadline is already elapsed"); + let error: WasmCommandError = + map_trap(wasmtime_error, WasmCommandError::Trap); + + assert!(matches!(error, WasmCommandError::DeadlineExceeded(_)), "{error}"); +} + +#[test] +fn map_trap_distinguishes_epoch_deadline_from_other_traps() { + let deadline_error: WasmCommandError = + map_trap(wasmtime::Error::from(wasmtime::Trap::Interrupt), WasmCommandError::Trap); + assert!(matches!(deadline_error, WasmCommandError::DeadlineExceeded(_))); + + let other_error: WasmCommandError = + map_trap(wasmtime::Error::from(wasmtime::Trap::OutOfFuel), WasmCommandError::Trap); + assert!(matches!(other_error, WasmCommandError::Trap(_))); +} + #[test] fn rejected_decide_errors_stay_rejections() { let error: WasmCommandError = map_decide_error(DecideError::Rejected(host::DomainError { code: "already-exists".to_string(), message: "schedule already exists".to_string(), + details: Vec::new(), })); let WasmCommandError::Rejected(detail) = error else { @@ -135,6 +197,7 @@ fn faulted_decide_errors_stay_faults() { map_decide_error(DecideError::Faulted(host::DomainError { code: "decode-failed".to_string(), message: "payload did not decode".to_string(), + details: Vec::new(), })); let WasmCommandError::Faulted(detail) = error else { diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/lib.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/lib.rs index 2319a3004..93fa159f3 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/lib.rs @@ -41,6 +41,29 @@ //! folds module identity into the snapshot id: see [`WasmSnapshotId`]. A //! module version bump changes the id every snapshot lookup uses, so an old //! snapshot is simply not found and execution falls back to a full replay. +//! +//! # Snapshot failure recovery +//! +//! A snapshot the host cannot trust, either because reading it failed or +//! because it claims a position past the stream's own history, is by default +//! a hard error, mirroring [`trogon_decider_runtime::execution::CommandExecution`]. +//! Callers that would rather discard the untrusted snapshot and replay from +//! the beginning can opt in with +//! [`WasmCommandExecution::with_snapshot_failure_policy`], passing +//! [`trogon_decider_runtime::DiscardAndReplaySnapshotFailure`] in place of the +//! default [`trogon_decider_runtime::FailOnSnapshotFailure`]. See +//! [`WasmSnapshotFailurePolicy`] and [`WasmSnapshotFailureContext`]. +//! +//! # Runtime rollout +//! +//! [`DeciderRegistry`] is immutable once built: swapping in a new module +//! version means building and installing a whole new registry. Callers that +//! need to change routes while commands are in flight, without dropping +//! executions already dispatched against the old routes, should build a +//! [`DeciderRegistryHandle`] instead (via [`DeciderRegistryBuilder::build_handle`] +//! or [`DeciderRegistryHandle::from`]). See its type-level docs for the +//! atomicity guarantee the handle provides and for what a version swap means +//! for a stream's snapshots. mod command_spec; mod command_type; @@ -54,6 +77,8 @@ mod opaque_snapshot; mod registry; mod snapshot_id; +#[cfg(test)] +mod test_doubles; #[cfg(test)] mod test_fixture; @@ -61,14 +86,20 @@ pub use command_spec::WasmCommandSpec; pub use command_type::{CommandType, CommandTypeError}; pub use domain_error_detail::DomainErrorDetail; pub use engine::{ - DEFAULT_FUEL_PER_CALL, DEFAULT_MAX_MEMORY_BYTES, WasmDeciderEngine, WasmEngineConfig, WasmEngineError, + DEFAULT_EPOCH_TICK_INTERVAL, DEFAULT_EPOCH_TICKS_PER_CALL, DEFAULT_FUEL_PER_CALL, DEFAULT_MAX_CONCURRENT_SESSIONS, + DEFAULT_MAX_INSTANCES_PER_SESSION, DEFAULT_MAX_MEMORIES_PER_SESSION, DEFAULT_MAX_MEMORY_BYTES, + DEFAULT_MAX_TABLE_ELEMENTS, DEFAULT_MAX_TABLES_PER_SESSION, WasmDeciderEngine, WasmEngineConfig, WasmEngineError, }; pub use execution::{ - WasmCommandError, WasmCommandExecution, WasmExecutionResult, WithSnapshotStore, WithoutSnapshotStore, + WasmCommandError, WasmCommandExecution, WasmExecutionResult, WasmSnapshotFailureContext, WasmSnapshotFailurePolicy, + WithSnapshotStore, WithoutSnapshotStore, }; pub use module::{InvalidDescriptorError, LoadWasmDeciderError, WasmDeciderModule}; pub use module_name::{ModuleName, ModuleNameError}; pub use module_version::{ModuleVersion, ModuleVersionError}; pub use opaque_snapshot::OpaqueSnapshotPayload; -pub use registry::{DeciderRegistry, DeciderRegistryBuilder, RegisterModuleError, UnknownCommandTypeError}; +pub use registry::{ + ActivatedRoute, DeciderRegistry, DeciderRegistryBuilder, DeciderRegistryHandle, RegisterModuleError, RetiredRoute, + RouteInfo, UnknownCommandTypeError, +}; pub use snapshot_id::WasmSnapshotId; diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/module.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/module.rs index 64954256d..60a83fe3e 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/module.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/module.rs @@ -15,24 +15,33 @@ pub enum LoadWasmDeciderError { /// [`Linker`] instead of shelling out to `wasm-tools`. #[error("component declares an import; wasm decider components must be zero-import")] ForbiddenImports { + /// Underlying wasmtime error raised instantiating against an empty [`Linker`]. #[source] source: wasmtime::Error, }, + /// The component's bytes are not a valid wasm component, or targeted a wasmtime feature + /// this engine was not configured with. #[error("failed to compile wasm component")] Compile { + /// Underlying wasmtime compilation error. #[source] source: wasmtime::Error, }, + /// Instantiating the pre-instantiated component against a fresh store failed. #[error("failed to instantiate wasm component to probe its descriptor")] Instantiate { + /// Underlying wasmtime instantiation error. #[source] source: wasmtime::Error, }, + /// The guest's `descriptor()` export trapped or exceeded its call budget. #[error("failed to call descriptor() on wasm component")] Descriptor { + /// Underlying wasmtime error from the guest call. #[source] source: wasmtime::Error, }, + /// The guest's declared [`ModuleDescriptor`] failed structural validation. #[error("wasm component descriptor is invalid")] InvalidDescriptor(#[from] InvalidDescriptorError), } @@ -40,14 +49,21 @@ pub enum LoadWasmDeciderError { /// A structurally invalid [`ModuleDescriptor`] reported by a guest component. #[derive(Debug, Error, PartialEq, Eq)] pub enum InvalidDescriptorError { + /// The descriptor's `name` field is not a well-formed [`ModuleName`]. #[error("wasm component descriptor has an invalid module name")] InvalidName(#[source] crate::ModuleNameError), + /// The descriptor's `version` field is not a well-formed [`ModuleVersion`]. #[error("wasm component descriptor has an invalid module version")] InvalidVersion(#[source] crate::ModuleVersionError), + /// One of the descriptor's declared commands has an invalid command type string. #[error("wasm component descriptor declares an invalid command type")] InvalidCommandType(#[source] crate::CommandTypeError), + /// The descriptor declares the same command type more than once. #[error("wasm component descriptor declares command type '{command_type}' more than once")] - DuplicateCommandType { command_type: CommandType }, + DuplicateCommandType { + /// The command type declared more than once. + command_type: CommandType, + }, } /// A compiled, load-time-validated WASM decider component. @@ -57,6 +73,7 @@ pub enum InvalidDescriptorError { /// [`DeciderPre`] for cheap per-command instantiation, and probes the /// guest's `descriptor()` export once so routing and validation do not need /// to spin up a guest session. +#[derive(Clone)] pub struct WasmDeciderModule { engine: WasmDeciderEngine, decider_pre: DeciderPre, @@ -121,13 +138,31 @@ impl WasmDeciderModule { } } +#[cfg(test)] +impl WasmDeciderModule { + /// Returns a copy of this module reporting a different declared version. + /// + /// The only compiled fixture available to this crate's tests always + /// reports one fixed identity, so registry and rollout tests that need a + /// second, distinct module version synthesize one from the same compiled + /// bytes rather than building a second fixture. + pub(crate) fn with_version(mut self, version: ModuleVersion) -> Self { + self.version = version; + self + } +} + fn probe_descriptor( engine: &WasmDeciderEngine, decider_pre: &DeciderPre, ) -> Result { let mut store = engine.new_store(); - store - .set_fuel(engine.config().fuel_per_call()) + engine + .arm_guest_call( + &mut store, + engine.config().fuel_per_call(), + engine.config().epoch_ticks_per_call(), + ) .map_err(|source| LoadWasmDeciderError::Instantiate { source })?; let bindings = decider_pre .instantiate(&mut store) diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry.rs index a1197903c..0970688cf 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry.rs @@ -1,9 +1,9 @@ use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, PoisonError, RwLock, RwLockWriteGuard}; use thiserror::Error; -use crate::{CommandType, ModuleName, WasmDeciderModule}; +use crate::{CommandType, ModuleName, ModuleVersion, WasmDeciderModule}; /// Failure registering a [`WasmDeciderModule`] into a [`DeciderRegistry`]. #[derive(Debug, Error, PartialEq, Eq)] @@ -11,8 +11,11 @@ use crate::{CommandType, ModuleName, WasmDeciderModule}; "command type '{command_type}' is already routed to module '{existing_module}'; module '{new_module}' cannot also claim it" )] pub struct RegisterModuleError { + /// The command type both modules claim. pub command_type: CommandType, + /// The module that already owns `command_type`. pub existing_module: ModuleName, + /// The module whose registration was rejected. pub new_module: ModuleName, } @@ -20,13 +23,46 @@ pub struct RegisterModuleError { #[derive(Debug, Error, PartialEq, Eq)] #[error("no wasm decider module is registered for command type '{command_type}'")] pub struct UnknownCommandTypeError { + /// The command type with no registered route. pub command_type: CommandType, } +/// One command type's current route: the identity of the module handling it. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct RouteInfo { + /// The routed command type. + pub command_type: CommandType, + /// Name of the module handling `command_type`. + pub module_name: ModuleName, + /// Version of the module handling `command_type`. + pub module_version: ModuleVersion, +} + +/// One command type's outcome from a [`DeciderRegistryHandle::activate`] call. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ActivatedRoute { + /// The command type that was (re)routed. + pub command_type: CommandType, + /// The module that owned this route immediately before the swap, if any. + pub previous_module: Option<(ModuleName, ModuleVersion)>, +} + +/// The route removed by a [`DeciderRegistryHandle::retire`] call. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RetiredRoute { + /// The command type whose route was removed. + pub command_type: CommandType, + /// Name of the module that had owned `command_type`. + pub module_name: ModuleName, + /// Version of the module that had owned `command_type`. + pub module_version: ModuleVersion, +} + /// Routes command types to the WASM decider module that declared them. /// /// Built once at startup via [`DeciderRegistryBuilder`] and shared read-only -/// across command executions. +/// across command executions. See [`DeciderRegistryHandle`] for a form of +/// this registry that can be updated while executions are in flight. #[derive(Default)] pub struct DeciderRegistry { modules_by_command: HashMap>, @@ -46,6 +82,19 @@ impl DeciderRegistry { command_type: command_type.clone(), }) } + + /// Lists every currently routed command type alongside the identity of + /// the module handling it. + pub fn routes(&self) -> Vec { + self.modules_by_command + .iter() + .map(|(command_type, module)| RouteInfo { + command_type: command_type.clone(), + module_name: module.name().clone(), + module_version: module.version().clone(), + }) + .collect() + } } /// Builder for [`DeciderRegistry`] that rejects command type collisions across modules. @@ -77,6 +126,174 @@ impl DeciderRegistryBuilder { modules_by_command: self.modules_by_command, } } + + /// Finishes building a [`DeciderRegistryHandle`] wrapping this registry, + /// ready for runtime rollout via [`DeciderRegistryHandle::activate`] and + /// [`DeciderRegistryHandle::retire`]. + pub fn build_handle(self) -> DeciderRegistryHandle { + DeciderRegistryHandle::new(self.build()) + } +} + +/// A [`DeciderRegistry`] behind a handle that can be updated while command +/// executions are in flight. +/// +/// This crate has no workspace dependency on a lock-free swap primitive such +/// as `arc-swap`, so this handle is built from a +/// `std::sync::RwLock>` instead, following the same +/// poison-recovering lock idiom already used elsewhere in this workspace. +/// +/// # Atomicity +/// +/// The handle holds one [`Arc`] behind a [`RwLock`]. Every +/// read ([`Self::snapshot`], [`Self::route`], [`Self::routes`]) clones that +/// `Arc` under a brief read lock and then works against its own owned clone, +/// independent of whatever [`Self::activate`] or [`Self::retire`] does next. +/// [`Self::activate`] and [`Self::retire`] each compute the entire next +/// routing table first and take the write lock just long enough to install +/// it with one assignment. A concurrent reader therefore always observes one +/// complete, self-consistent [`DeciderRegistry`]: either every route the last +/// completed write installed, or every route the write before it installed, +/// never a mix of the two. In particular, a module that declares more than +/// one command type is activated or retired as a unit: a reader can never +/// see one of its command types already pointing at a new module version +/// while a sibling command type from that same module still points at the +/// old one. Lock poisoning from a panicked writer is recovered rather than +/// propagated (`RwLock::read`/`write().unwrap_or_else(PoisonError::into_inner)`), +/// so a prior panic elsewhere can never turn a routing lookup into a panic +/// of its own. +/// +/// # Dispatch must pin its own resolution +/// +/// [`Self::activate`] and [`Self::retire`] only ever affect *future* calls to +/// [`Self::route`] and [`Self::snapshot`]. A caller must resolve the module +/// once per command dispatch and keep using that resolved +/// `Arc` (or `Arc`, from +/// [`Self::snapshot`]) for the rest of that command's execution. Nothing +/// about a later swap reaches back into an execution already running +/// against a module it resolved earlier: the `Arc` it is holding keeps that +/// exact module alive and unchanged no matter how many times the handle is +/// swapped afterward. +/// +/// # Snapshot behavior across a version swap +/// +/// [`crate::WasmSnapshotId`] folds a module's name and version into every +/// snapshot id it produces (`{module_name}@{module_version}/{stream_id}`). +/// Rerouting a command type from module version `v1` to `v2` via +/// [`Self::activate`] does not migrate or invalidate `v1`'s snapshots: `v2`'s +/// snapshot id for a given stream is simply a different string. The first +/// command `v2` executes against a stream `v1` had already snapshotted reads +/// the snapshot store under `v2`'s id, finds nothing, and falls back to a +/// full replay of that stream's history from the beginning (the same +/// fallback `execution::WasmCommandExecution` already takes for a stream +/// that was never snapshotted at all). That replay reconstructs the exact +/// same guest state a snapshot would have, only cold: every prior event is +/// read and replayed once more instead of resuming from a saved position. +/// `v2` then writes its own snapshot under its own id, so `v1`'s snapshot is +/// left untouched; routing back to `v1` later finds that snapshot exactly +/// where it left it. No migration step, explicit invalidation, or +/// coordination between versions is required: the version bump folded into +/// the snapshot id is enough on its own. See the `activating_a_new_module_version_*` +/// test in this module's `tests` submodule for this behavior exercised end +/// to end. +pub struct DeciderRegistryHandle { + current: RwLock>, +} + +impl DeciderRegistryHandle { + /// Wraps an existing [`DeciderRegistry`] in a swappable handle. + pub fn new(registry: DeciderRegistry) -> Self { + Self { + current: RwLock::new(Arc::new(registry)), + } + } + + /// Returns the [`DeciderRegistry`] this handle currently points at. + /// + /// Hold the returned `Arc` for the lifetime of one command dispatch so a + /// concurrent [`Self::activate`] or [`Self::retire`] cannot change which + /// module that dispatch resolves to partway through. + pub fn snapshot(&self) -> Arc { + self.read() + } + + /// Looks up the module currently responsible for the given command type. + /// + /// Returns an owned `Arc`, independent of any later [`Self::activate`] or + /// [`Self::retire`] call, so the caller can keep executing against + /// exactly the module it resolved here. + pub fn route(&self, command_type: &CommandType) -> Result, UnknownCommandTypeError> { + self.read().route(command_type).map(Arc::clone) + } + + /// Lists every currently routed command type alongside the identity of + /// the module handling it. + pub fn routes(&self) -> Vec { + self.read().routes() + } + + /// Routes every command type `module` declares to `module`, replacing + /// whatever module previously owned each of those routes. + /// + /// See the type-level docs for what this means for in-flight executions + /// and for a stream's snapshots taken under a route's previous module + /// version. + pub fn activate(&self, module: WasmDeciderModule) -> Vec { + let module = Arc::new(module); + let command_types = module.command_types().cloned().collect::>(); + + let mut current = self.write(); + let mut modules_by_command = current.modules_by_command.clone(); + let outcomes = command_types + .into_iter() + .map(|command_type| { + let previous = modules_by_command.insert(command_type.clone(), Arc::clone(&module)); + ActivatedRoute { + command_type, + previous_module: previous.map(|previous| (previous.name().clone(), previous.version().clone())), + } + }) + .collect(); + *current = Arc::new(DeciderRegistry { modules_by_command }); + outcomes + } + + /// Removes the route for one command type, if any module currently owns it. + /// + /// A command dispatched for `command_type` after this call fails with + /// [`UnknownCommandTypeError`] until a later [`Self::activate`] routes it + /// again. + pub fn retire(&self, command_type: &CommandType) -> Option { + let mut current = self.write(); + let mut modules_by_command = current.modules_by_command.clone(); + let removed = modules_by_command.remove(command_type)?; + *current = Arc::new(DeciderRegistry { modules_by_command }); + Some(RetiredRoute { + command_type: command_type.clone(), + module_name: removed.name().clone(), + module_version: removed.version().clone(), + }) + } + + fn read(&self) -> Arc { + Arc::clone(&self.current.read().unwrap_or_else(PoisonError::into_inner)) + } + + fn write(&self) -> RwLockWriteGuard<'_, Arc> { + self.current.write().unwrap_or_else(PoisonError::into_inner) + } +} + +impl Default for DeciderRegistryHandle { + fn default() -> Self { + Self::new(DeciderRegistry::default()) + } +} + +impl From for DeciderRegistryHandle { + fn from(registry: DeciderRegistry) -> Self { + Self::new(registry) + } } #[cfg(test)] diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry/tests.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry/tests.rs index 1bacc4ec5..5ee831f96 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry/tests.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/registry/tests.rs @@ -1,12 +1,84 @@ +use buffa::Message as _; +use buffa::MessageField; +use trogon_decider_runtime::{ImmediateSnapshotTaskScheduler, ReadFrom, StreamPosition}; +use trogon_decider_wit::host::CommandEnvelope; +use trogonai_proto::content::v1alpha1 as content_v1alpha1; +use trogonai_proto::scheduler::schedules::v1; + use super::*; +use crate::test_doubles::{InMemoryEventStore, InMemorySnapshotStore}; use crate::test_fixture::schedules_bytes; -use crate::{WasmDeciderEngine, WasmEngineConfig}; +use crate::{WasmCommandExecution, WasmDeciderEngine, WasmEngineConfig, WasmSnapshotId}; fn schedules_module() -> WasmDeciderModule { let engine = WasmDeciderEngine::new(WasmEngineConfig::default()).expect("engine builds"); WasmDeciderModule::load(engine, &schedules_bytes()).expect("module loads") } +fn create_type() -> CommandType { + CommandType::new(trogonai_proto::scheduler::schedules::CREATE_SCHEDULE_TYPE_URL).expect("valid command type") +} + +fn pause_type() -> CommandType { + CommandType::new(trogonai_proto::scheduler::schedules::PAUSE_SCHEDULE_TYPE_URL).expect("valid command type") +} + +fn create_command(id: &str) -> CommandEnvelope { + CommandEnvelope { + type_: trogonai_proto::scheduler::schedules::CREATE_SCHEDULE_TYPE_URL.to_string(), + payload: v1::CreateSchedule { + schedule_id: id.to_string(), + status: MessageField::some(v1::ScheduleStatus { + kind: Some(v1::schedule_status::Scheduled {}.into()), + }), + schedule: MessageField::some(v1::Schedule { + kind: Some( + v1::schedule::Every { + every: MessageField::some(buffa_types::google::protobuf::Duration { + seconds: 30, + nanos: 0, + ..buffa_types::google::protobuf::Duration::default() + }), + } + .into(), + ), + }), + delivery: MessageField::some(v1::Delivery { + kind: Some( + v1::delivery::NatsMessage { + subject: "agent.run".to_string(), + ttl: MessageField::none(), + source: MessageField::none(), + } + .into(), + ), + }), + message: MessageField::some(v1::Message { + content: MessageField::some(content_v1alpha1::Content { + content_type: "application/json".to_string(), + data: br#"{"kind":"heartbeat"}"#.to_vec(), + }), + headers: Vec::new(), + }), + } + .encode_to_vec(), + } +} + +fn pause_command(id: &str) -> CommandEnvelope { + CommandEnvelope { + type_: trogonai_proto::scheduler::schedules::PAUSE_SCHEDULE_TYPE_URL.to_string(), + payload: v1::PauseSchedule { + schedule_id: id.to_string(), + } + .encode_to_vec(), + } +} + +fn position(value: u64) -> StreamPosition { + StreamPosition::try_new(value).expect("test stream position must be non-zero") +} + #[test] fn routes_registered_command_types() { let registry = DeciderRegistry::builder() @@ -46,3 +118,203 @@ fn duplicate_command_type_across_modules_is_rejected() { assert_eq!(error.existing_module.as_str(), "scheduler.schedules"); assert_eq!(error.new_module.as_str(), "scheduler.schedules"); } + +#[test] +fn routes_reports_every_registered_command_type_with_module_identity() { + let registry = DeciderRegistry::builder() + .register(schedules_module()) + .expect("registration succeeds") + .build(); + + let mut routes = registry.routes(); + routes.sort(); + + assert_eq!(routes.len(), 4); + assert!( + routes + .iter() + .all(|route| route.module_name.as_str() == "scheduler.schedules") + ); + assert!(routes.iter().any(|route| route.command_type == create_type())); + assert!(routes.iter().any(|route| route.command_type == pause_type())); +} + +#[test] +fn handle_routes_delegate_to_the_wrapped_registry() { + let registry = DeciderRegistry::builder() + .register(schedules_module()) + .expect("registration succeeds") + .build(); + let mut expected_routes = registry.routes(); + expected_routes.sort(); + + let handle = DeciderRegistryHandle::from(registry); + + let mut routes = handle.routes(); + routes.sort(); + assert_eq!(routes, expected_routes); + + let module = handle.route(&create_type()).expect("route succeeds"); + assert_eq!(module.name().as_str(), "scheduler.schedules"); + assert_eq!(handle.snapshot().routes().len(), 4); +} + +#[test] +fn activate_adds_routes_for_every_command_type_the_module_declares() { + let handle = DeciderRegistryHandle::default(); + + let activated = handle.activate(schedules_module()); + + assert_eq!(activated.len(), 4); + assert!(activated.iter().all(|route| route.previous_module.is_none())); + assert!(activated.iter().any(|route| route.command_type == create_type())); + assert!(activated.iter().any(|route| route.command_type == pause_type())); + + let module = handle.route(&create_type()).expect("route succeeds"); + assert_eq!(module.name().as_str(), "scheduler.schedules"); +} + +#[test] +fn activate_replaces_the_current_route_and_reports_the_previous_module_identity() { + let handle = DeciderRegistryHandle::default(); + let module_v1 = schedules_module(); + let v1_version = module_v1.version().clone(); + handle.activate(module_v1.clone()); + + let module_v2 = module_v1.with_version(ModuleVersion::new("0.2.0").expect("valid version")); + let activated = handle.activate(module_v2); + + assert_eq!(activated.len(), 4); + for route in &activated { + let previous = route + .previous_module + .as_ref() + .expect("a previous module owned this route"); + assert_eq!(previous.0.as_str(), "scheduler.schedules"); + assert_eq!(previous.1, v1_version); + } + + let module = handle.route(&create_type()).expect("route succeeds"); + assert_eq!(module.version().as_str(), "0.2.0"); +} + +#[test] +fn retire_removes_the_route_and_reports_the_retired_module_identity() { + let handle = DeciderRegistryHandle::default(); + handle.activate(schedules_module()); + + let retired = handle.retire(&create_type()).expect("create was routed"); + assert_eq!(retired.command_type, create_type()); + assert_eq!(retired.module_name.as_str(), "scheduler.schedules"); + + assert!(handle.route(&create_type()).is_err()); + assert!(handle.route(&pause_type()).is_ok()); + assert!(handle.retire(&create_type()).is_none()); +} + +#[test] +fn concurrent_activation_never_exposes_a_torn_route_pair() { + let module_v1 = schedules_module(); + let module_v2 = module_v1 + .clone() + .with_version(ModuleVersion::new("0.2.0").expect("valid version")); + + let handle = DeciderRegistryHandle::default(); + handle.activate(module_v1.clone()); + + let handle_ref = &handle; + std::thread::scope(|scope| { + scope.spawn(move || { + for _ in 0..200 { + handle_ref.activate(module_v1.clone()); + handle_ref.activate(module_v2.clone()); + } + }); + + for _ in 0..200 { + // Pinning one snapshot before reading both routes is essential here: two + // separate `handle.route()` calls each take their own read lock, so a + // swap between them could legitimately observe two different versions + // without that meaning the swap itself was torn. Resolving through one + // `Arc` is what this test needs to exercise. + let registry = handle.snapshot(); + let create_version = registry + .route(&create_type()) + .expect("route succeeds") + .version() + .clone(); + let pause_version = registry.route(&pause_type()).expect("route succeeds").version().clone(); + assert_eq!( + create_version, pause_version, + "a single resolved registry snapshot must never disagree with itself about which module \ + version owns two command types declared by the same module" + ); + } + }); +} + +#[tokio::test] +async fn activating_a_new_module_version_starts_cold_and_keeps_the_prior_snapshot() { + let module_v1 = schedules_module(); + let module_v2 = module_v1 + .clone() + .with_version(ModuleVersion::new("0.2.0").expect("valid version")); + + let handle = DeciderRegistry::builder() + .register(module_v1) + .expect("registration succeeds") + .build_handle(); + + let event_store = InMemoryEventStore::default(); + let snapshot_store = InMemorySnapshotStore::default(); + let scheduler = ImmediateSnapshotTaskScheduler; + + let v1_create_module = handle.route(&create_type()).expect("v1 routes create"); + WasmCommandExecution::new(&v1_create_module, &event_store, &create_command("backup")) + .with_snapshot_store(&snapshot_store, &scheduler) + .execute() + .await + .expect("create succeeds against v1"); + + let v1_snapshot_id = WasmSnapshotId::new(v1_create_module.name(), v1_create_module.version(), "backup"); + assert_eq!( + snapshot_store + .get(v1_snapshot_id.as_str()) + .expect("v1 must have written a snapshot") + .position, + position(1) + ); + + let activated = handle.activate(module_v2); + assert_eq!(activated.len(), 4); + + let v2_pause_module = handle.route(&pause_type()).expect("v2 now routes pause"); + assert_eq!(v2_pause_module.version().as_str(), "0.2.0"); + + let result = WasmCommandExecution::new(&v2_pause_module, &event_store, &pause_command("backup")) + .with_snapshot_store(&snapshot_store, &scheduler) + .execute() + .await + .expect("pause against v2 replays from the beginning instead of resuming a v1 snapshot"); + + assert_eq!(result.stream_position, position(2)); + assert_eq!( + event_store.reads_from(), + vec![ReadFrom::Beginning], + "v2 has no snapshot of its own yet, so it must do a full replay rather than resume v1's snapshot position" + ); + + let v2_snapshot_id = WasmSnapshotId::new(v2_pause_module.name(), v2_pause_module.version(), "backup"); + assert_ne!(v1_snapshot_id, v2_snapshot_id); + assert!( + snapshot_store.get(v1_snapshot_id.as_str()).is_some(), + "v1's snapshot must survive the version swap untouched" + ); + assert_eq!( + snapshot_store + .get(v2_snapshot_id.as_str()) + .expect("v2 must write its own snapshot side by side with v1's") + .position, + position(2) + ); +} diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/src/test_doubles.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/src/test_doubles.rs new file mode 100644 index 000000000..c8e2f3d96 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/src/test_doubles.rs @@ -0,0 +1,149 @@ +//! In-memory storage doubles shared by this crate's own unit tests. +//! +//! Integration tests under `tests/` maintain their own copy of equivalent +//! doubles (they cannot see crate-private items), following the same +//! convention documented on [`crate::test_fixture`]. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; + +use trogon_decider_runtime::{ + AppendStreamRequest, AppendStreamResponse, ReadFrom, ReadSnapshotRequest, ReadSnapshotResponse, ReadStreamRequest, + ReadStreamResponse, Snapshot, SnapshotRead, SnapshotWrite, StreamAppend, StreamEvent, StreamPosition, StreamRead, + StreamWritePrecondition, WriteSnapshotRequest, WriteSnapshotResponse, +}; + +use crate::OpaqueSnapshotPayload; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +pub(crate) enum InfraError { + #[error("append rejected by write precondition")] + PreconditionFailed, +} + +#[derive(Default)] +struct EventStoreState { + events: Vec, + reads_from: Vec, +} + +/// Records every read so tests can assert on replay behavior, not only on +/// the outcome of an execution. +#[derive(Default)] +pub(crate) struct InMemoryEventStore { + state: Mutex, +} + +impl InMemoryEventStore { + pub(crate) fn reads_from(&self) -> Vec { + self.lock().reads_from.clone() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, EventStoreState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn current_position(state: &EventStoreState, stream_id: &str) -> Option { + state + .events + .iter() + .filter(|event| event.stream_id == stream_id) + .map(|event| event.stream_position) + .max() + } +} + +impl StreamRead for InMemoryEventStore { + type Error = InfraError; + + async fn read_stream(&self, request: ReadStreamRequest<'_, str>) -> Result { + let mut state = self.lock(); + state.reads_from.push(request.from); + let from_sequence = match request.from { + ReadFrom::Beginning => 1, + ReadFrom::Position(position) => position.as_u64(), + }; + Ok(ReadStreamResponse { + current_position: Self::current_position(&state, request.stream_id), + events: state + .events + .iter() + .filter(|event| event.stream_id == request.stream_id) + .filter(|event| event.stream_position.as_u64() >= from_sequence) + .cloned() + .collect(), + }) + } +} + +impl StreamAppend for InMemoryEventStore { + type Error = InfraError; + + async fn append_stream(&self, request: AppendStreamRequest<'_, str>) -> Result { + let mut state = self.lock(); + let current_position = Self::current_position(&state, request.stream_id); + match request.stream_write_precondition { + StreamWritePrecondition::Any => {} + StreamWritePrecondition::StreamExists if current_position.is_some() => {} + StreamWritePrecondition::NoStream if current_position.is_none() => {} + StreamWritePrecondition::At(position) if current_position == Some(position) => {} + _ => return Err(InfraError::PreconditionFailed), + } + + let mut next_sequence = current_position.map(StreamPosition::as_u64).unwrap_or(0); + for event in request.events { + next_sequence += 1; + state.events.push(StreamEvent { + stream_id: request.stream_id.to_string(), + event, + stream_position: StreamPosition::try_new(next_sequence).expect("sequence starts at one"), + recorded_at: chrono::Utc::now(), + }); + } + + Ok(AppendStreamResponse { + stream_position: StreamPosition::try_new(next_sequence).expect("append stores at least one event"), + }) + } +} + +/// Snapshot store double keyed by the caller-supplied snapshot id. +#[derive(Clone, Default)] +pub(crate) struct InMemorySnapshotStore { + snapshots: Arc>>>, +} + +impl InMemorySnapshotStore { + pub(crate) fn get(&self, snapshot_id: &str) -> Option> { + self.lock().get(snapshot_id).cloned() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, HashMap>> { + self.snapshots.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +impl SnapshotRead for InMemorySnapshotStore { + type Error = InfraError; + + async fn read_snapshot( + &self, + request: ReadSnapshotRequest<'_, str>, + ) -> Result, Self::Error> { + Ok(ReadSnapshotResponse { + snapshot: self.get(request.snapshot_id), + }) + } +} + +impl SnapshotWrite for InMemorySnapshotStore { + type Error = InfraError; + + async fn write_snapshot( + &self, + request: WriteSnapshotRequest<'_, OpaqueSnapshotPayload, str>, + ) -> Result { + self.lock().insert(request.snapshot_id.to_string(), request.snapshot); + Ok(WriteSnapshotResponse) + } +} diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/tests/schedules_execution.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/tests/schedules_execution.rs index 43d4bc53b..0b2f6cc95 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/tests/schedules_execution.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/tests/schedules_execution.rs @@ -10,7 +10,9 @@ use buffa::Message as _; use buffa::MessageField; use buffa::MessageName as _; use support::{InMemoryEventStore, InMemorySnapshotStore}; -use trogon_decider_runtime::{ImmediateSnapshotTaskScheduler, ReadFrom, StreamPosition, StreamWritePrecondition}; +use trogon_decider_runtime::{ + DiscardAndReplaySnapshotFailure, ImmediateSnapshotTaskScheduler, ReadFrom, StreamPosition, StreamWritePrecondition, +}; use trogon_decider_wasm_runtime::{ OpaqueSnapshotPayload, WasmCommandError, WasmCommandExecution, WasmDeciderEngine, WasmDeciderModule, WasmEngineConfig, WasmSnapshotId, @@ -237,6 +239,96 @@ async fn a_snapshot_ahead_of_the_stream_is_rejected() { assert!(matches!(error, WasmCommandError::SnapshotAheadOfStream(_)), "{error}"); } +#[tokio::test] +async fn a_snapshot_read_failure_is_rejected_by_default() { + let module = schedules_module(); + let event_store = InMemoryEventStore::default(); + let snapshot_store = InMemorySnapshotStore::default(); + let scheduler = ImmediateSnapshotTaskScheduler; + + WasmCommandExecution::new(&module, &event_store, &create_command("backup")) + .execute() + .await + .expect("create succeeds"); + snapshot_store.fail_reads(); + + let Err(error) = WasmCommandExecution::new(&module, &event_store, &pause_command("backup")) + .with_snapshot_store(&snapshot_store, &scheduler) + .execute() + .await + else { + panic!("expected snapshot read failure"); + }; + + assert!(matches!(error, WasmCommandError::ReadSnapshot(_)), "{error}"); + assert_eq!(event_store.read_stream_calls(), 0); +} + +#[tokio::test] +async fn discard_and_replay_recovers_from_a_snapshot_read_failure() { + let module = schedules_module(); + let event_store = InMemoryEventStore::default(); + let snapshot_store = InMemorySnapshotStore::default(); + let scheduler = ImmediateSnapshotTaskScheduler; + + WasmCommandExecution::new(&module, &event_store, &create_command("backup")) + .execute() + .await + .expect("create succeeds"); + snapshot_store.fail_reads(); + + let result = WasmCommandExecution::new(&module, &event_store, &pause_command("backup")) + .with_snapshot_store(&snapshot_store, &scheduler) + .with_snapshot_failure_policy(DiscardAndReplaySnapshotFailure) + .execute() + .await + .expect("discard-and-replay recovers from the unreadable snapshot"); + + assert_eq!(result.stream_position, position(2)); + assert_eq!(event_store.reads_from(), vec![ReadFrom::Beginning]); + + let snapshot_id = WasmSnapshotId::new(module.name(), module.version(), "backup"); + let snapshot = snapshot_store + .get(snapshot_id.as_str()) + .expect("a fresh snapshot replaces the unreadable one"); + assert_eq!(snapshot.position, position(2)); +} + +#[tokio::test] +async fn discard_and_replay_recovers_from_a_snapshot_ahead_of_stream() { + let module = schedules_module(); + let event_store = InMemoryEventStore::default(); + let snapshot_store = InMemorySnapshotStore::default(); + let scheduler = ImmediateSnapshotTaskScheduler; + + WasmCommandExecution::new(&module, &event_store, &create_command("backup")) + .execute() + .await + .expect("create succeeds"); + + let snapshot_id = WasmSnapshotId::new(module.name(), module.version(), "backup"); + snapshot_store.insert( + snapshot_id.as_str(), + trogon_decider_runtime::Snapshot::new(position(5), OpaqueSnapshotPayload::new(Vec::new())), + ); + + let result = WasmCommandExecution::new(&module, &event_store, &pause_command("backup")) + .with_snapshot_store(&snapshot_store, &scheduler) + .with_snapshot_failure_policy(DiscardAndReplaySnapshotFailure) + .execute() + .await + .expect("discard-and-replay recovers from the ahead-of-stream snapshot"); + + assert_eq!(result.stream_position, position(2)); + let stale_resume = ReadFrom::after(position(5)).expect("resume position advances"); + assert_eq!(event_store.reads_from(), vec![stale_resume, ReadFrom::Beginning]); + + let refreshed = snapshot_store + .get(snapshot_id.as_str()) + .expect("a fresh snapshot replaces the ahead-of-stream one"); + assert_eq!(refreshed.position, position(2)); +} + #[tokio::test] async fn builder_overrides_shape_the_appended_events() { #[derive(Debug, Clone, Copy)] @@ -327,3 +419,29 @@ fn an_exhausted_fuel_budget_fails_the_load_probe() { let engine = WasmDeciderEngine::new(WasmEngineConfig::default().with_fuel_per_call(1)).expect("engine builds"); assert!(WasmDeciderModule::load(engine, &schedules_wasm()).is_err()); } + +#[tokio::test] +async fn the_pooling_allocator_engine_executes_the_fixture_end_to_end() { + let engine = WasmDeciderEngine::new( + WasmEngineConfig::default() + .with_max_concurrent_sessions(2) + .with_max_instances_per_session(3) + .with_max_tables_per_session(2) + .with_max_memories_per_session(1), + ) + .expect("pooling-allocator engine builds"); + let module = WasmDeciderModule::load(engine, &schedules_wasm()).expect("module loads under the pooling allocator"); + let event_store = InMemoryEventStore::default(); + + let create_result = WasmCommandExecution::new(&module, &event_store, &create_command("backup")) + .execute() + .await + .expect("create succeeds under the pooling allocator"); + assert_eq!(create_result.stream_position, position(1)); + + let pause_result = WasmCommandExecution::new(&module, &event_store, &pause_command("backup")) + .execute() + .await + .expect("pause succeeds under the pooling allocator"); + assert_eq!(pause_result.stream_position, position(2)); +} diff --git a/rsworkspace/crates/trogon-decider-wasm-runtime/tests/support/mod.rs b/rsworkspace/crates/trogon-decider-wasm-runtime/tests/support/mod.rs index 158aec823..93929bd2b 100644 --- a/rsworkspace/crates/trogon-decider-wasm-runtime/tests/support/mod.rs +++ b/rsworkspace/crates/trogon-decider-wasm-runtime/tests/support/mod.rs @@ -13,9 +13,11 @@ use trogon_decider_wasm_runtime::OpaqueSnapshotPayload; #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] pub enum InfraError { #[error("append rejected by write precondition")] - PreconditionFailed, + PreconditionRejected, #[error("snapshot write failed")] SnapshotWriteFailed, + #[error("snapshot read failed")] + SnapshotReadFailed, } #[derive(Default)] @@ -112,7 +114,7 @@ impl StreamAppend for InMemoryEventStore { StreamWritePrecondition::StreamExists if current_position.is_some() => {} StreamWritePrecondition::NoStream if current_position.is_none() => {} StreamWritePrecondition::At(position) if current_position == Some(position) => {} - _ => return Err(InfraError::PreconditionFailed), + _ => return Err(InfraError::PreconditionRejected), } let mut next_sequence = current_position.map(StreamPosition::as_u64).unwrap_or(0); @@ -137,6 +139,7 @@ impl StreamAppend for InMemoryEventStore { pub struct InMemorySnapshotStore { snapshots: Arc>>>, fail_writes: Arc, + fail_reads: Arc, } impl InMemorySnapshotStore { @@ -148,6 +151,10 @@ impl InMemorySnapshotStore { self.fail_writes.store(true, std::sync::atomic::Ordering::SeqCst); } + pub fn fail_reads(&self) { + self.fail_reads.store(true, std::sync::atomic::Ordering::SeqCst); + } + pub fn get(&self, snapshot_id: &str) -> Option> { self.lock().get(snapshot_id).cloned() } @@ -164,6 +171,9 @@ impl SnapshotRead for InMemorySnapshotStore { &self, request: ReadSnapshotRequest<'_, str>, ) -> Result, Self::Error> { + if self.fail_reads.load(std::sync::atomic::Ordering::SeqCst) { + return Err(InfraError::SnapshotReadFailed); + } Ok(ReadSnapshotResponse { snapshot: self.get(request.snapshot_id), }) diff --git a/rsworkspace/crates/trogon-decider-wit/Cargo.toml b/rsworkspace/crates/trogon-decider-wit/Cargo.toml index b1892da9e..0455cca40 100644 --- a/rsworkspace/crates/trogon-decider-wit/Cargo.toml +++ b/rsworkspace/crates/trogon-decider-wit/Cargo.toml @@ -11,8 +11,16 @@ default = [] guest = ["dep:wit-bindgen"] host = ["dep:wasmtime", "dep:anyhow"] -[lints] -workspace = true +[lints.rust] +missing_docs = "warn" +warnings = "deny" +unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(dylint_lib, values(any()))'] } + +[lints.clippy] +all = "deny" +expect_used = "deny" +panic = "deny" +unwrap_used = "deny" [dependencies] anyhow = { workspace = true, optional = true } diff --git a/rsworkspace/crates/trogon-decider-wit/README.md b/rsworkspace/crates/trogon-decider-wit/README.md new file mode 100644 index 000000000..ba066ef68 --- /dev/null +++ b/rsworkspace/crates/trogon-decider-wit/README.md @@ -0,0 +1,19 @@ +# trogon-decider-wit + +Per [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md), this crate owns the +`trogon:decider` WIT contract (`wit/world.wit`) as generated Rust bindings, compiled for both +sides that implement it: `guest` (feature `guest`, `wit_bindgen::generate!`, for `wasm32` +components) and `host` (feature `host`, `wasmtime::component::bindgen!`, for non-`wasm32` +runtimes that load and drive them). It implements neither side itself; it exists so both a +guest component and a host runtime compile against exactly the same generated types +(`AnyEnvelope`, `CommandEnvelope`, `CommandSpec`, `DecideError`, `DomainError`, `Guest`, +`GuestSession`, `ModuleDescriptor`, `WritePrecondition`) rather than hand-kept mirrors of each +other. + +`WIT_DIR`, an absolute path to this crate's own `wit/` directory baked in at this crate's +compile time, is what lets `export_decider!` (in `trogon-decider-guest-macros`) resolve the WIT +contract regardless of the calling component crate's location in the workspace. + +See `wit/world.wit` for the interface itself, and +[Decider Platform](../../../docs/architecture/decider.md) for how the `handler` interface's +`descriptor`/`stream-id`/`session` shape maps onto the native `Decider` trait. diff --git a/rsworkspace/crates/trogon-decider-wit/src/lib.rs b/rsworkspace/crates/trogon-decider-wit/src/lib.rs index 7e6ffecbc..ce73d0f12 100644 --- a/rsworkspace/crates/trogon-decider-wit/src/lib.rs +++ b/rsworkspace/crates/trogon-decider-wit/src/lib.rs @@ -1,89 +1,153 @@ +//! WIT contract and generated bindings shared by native hosts and WASM guest deciders. +//! +//! [`world.wit`](https://github.com/TrogonStack/trogonai) defines the `handler` interface every +//! compiled decider component exports: a `descriptor()` query, a `stream-id()` resolver, and a +//! `session` resource with `evolve`/`decide`/`snapshot` methods. This crate does not implement +//! that contract; it only compiles it into Rust bindings for the two sides that do: +//! +//! - [`guest`] (feature `guest`), generated with `wit_bindgen::generate!` for `wasm32` components. +//! - [`host`] (feature `host`, non-`wasm32` only), generated with `wasmtime::component::bindgen!` +//! for the runtimes that load and drive those components. +//! +//! Both sides re-export the same WIT-derived types (`AnyEnvelope`, `CommandEnvelope`, +//! `CommandSpec`, `DecideError`, `DomainError`, `Guest`, `GuestSession`, `ModuleDescriptor`, +//! `WritePrecondition`) so application code can reason about one contract regardless of which +//! side it is compiled into. + +/// Absolute path to this crate's `wit/` directory. +/// +/// `wit_bindgen::generate!` and `wasmtime::component::bindgen!`'s `path:` field resolve +/// relative to the manifest directory of whichever crate is compiling when the macro expands, +/// not this crate's. The `export_decider!` macro (in `trogon-decider-guest-macros`) reads this +/// constant at macro-expansion time and interpolates it as an absolute path, so a component +/// crate can live at any directory depth relative to `trogon-decider-wit`. +pub const WIT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/wit"); + +/// Guest-side bindings, generated for `wasm32` components with `wit_bindgen::generate!`. +/// +/// Implement [`Guest`] and [`GuestSession`] to satisfy the `handler` world; `export_decider!` +/// (in `trogon-decider-guest-macros`) wires a [`Decider`](trogon_decider::Decider) implementation +/// to these traits. The `Guest`/`GuestSession` traits and `Guest::Session` associated type are +/// synthesized wholesale by `wit_bindgen` from the `handler` interface's exports and carry no +/// individual doc anchor in `world.wit`; see the WIT source and this module's re-exported types +/// for their contract. #[cfg(feature = "guest")] #[cfg_attr(dylint_lib = "trogon_lints", allow(inline_module_block))] pub mod guest { - wit_bindgen::generate!({ - world: "decider", - path: "wit", - generate_all, - }); + mod bindings { + #![allow(missing_docs, reason = "generated bindings; see the module doc and world.wit")] + + wit_bindgen::generate!({ + world: "decider", + path: "wit", + generate_all, + }); + } - pub use exports::trogon::decider::handler::{ + pub use bindings::exports::trogon::decider::handler::{ AnyEnvelope, CommandEnvelope, CommandSpec, DecideError, DomainError, Guest, GuestSession, ModuleDescriptor, WritePrecondition, }; } +/// Host-side bindings, generated for non-`wasm32` runtimes with `wasmtime::component::bindgen!`. +/// +/// [`trogon_decider_wasm_runtime`](https://docs.rs/trogon-decider-wasm-runtime) drives a +/// compiled component through the thin wrapper functions below rather than calling the +/// generated `Decider` bindings struct directly, so the execution boundary stays a small, +/// reviewable surface independent of `wasmtime-component-macro`'s generated shape. #[cfg(all(feature = "host", not(target_arch = "wasm32")))] #[cfg_attr(dylint_lib = "trogon_lints", allow(inline_module_block))] pub mod host { use wasmtime::component::ResourceAny; - wasmtime::component::bindgen!({ - world: "decider", - path: "wit", - }); + /// Handle to a guest `session` resource, opaque to the host beyond its lifecycle. + pub type Session = ResourceAny; - pub use exports::trogon::decider::handler::{ - AnyEnvelope, CommandEnvelope, CommandSpec, DecideError, DomainError, Guest, GuestSession, ModuleDescriptor, - WritePrecondition, - }; + mod bindings { + #![allow(missing_docs, reason = "generated bindings; see the module doc and world.wit")] - pub type Session = ResourceAny; + use super::Session; - pub fn call_descriptor( - bindings: &Decider, - store: &mut wasmtime::Store, - ) -> wasmtime::Result { - bindings.interface0.call_descriptor(store) - } + wasmtime::component::bindgen!({ + world: "decider", + path: "wit", + }); - pub fn call_stream_id( - bindings: &Decider, - store: &mut wasmtime::Store, - command: &CommandEnvelope, - ) -> wasmtime::Result> { - bindings.interface0.call_stream_id(store, command) - } + use exports::trogon::decider::handler::{ + AnyEnvelope, CommandEnvelope, DecideError, DomainError, ModuleDescriptor, + }; - pub fn create_session( - bindings: &Decider, - store: &mut wasmtime::Store, - snapshot: Option<&[u8]>, - ) -> wasmtime::Result { - bindings.interface0.session().call_constructor(store, snapshot) - } + /// Calls the guest's `descriptor()` export. + pub fn call_descriptor( + bindings: &Decider, + store: &mut wasmtime::Store, + ) -> wasmtime::Result { + bindings.interface0.call_descriptor(store) + } - pub fn evolve( - bindings: &Decider, - store: &mut wasmtime::Store, - session: Session, - events: &[AnyEnvelope], - ) -> wasmtime::Result> { - bindings.interface0.session().call_evolve(store, session, events) - } + /// Calls the guest's `stream-id()` export. + pub fn call_stream_id( + bindings: &Decider, + store: &mut wasmtime::Store, + command: &CommandEnvelope, + ) -> wasmtime::Result> { + bindings.interface0.call_stream_id(store, command) + } - pub fn decide( - bindings: &Decider, - store: &mut wasmtime::Store, - session: Session, - command: &CommandEnvelope, - ) -> wasmtime::Result, DecideError>> { - bindings.interface0.session().call_decide(store, session, command) - } + /// Constructs a new guest `session` resource, optionally resuming from a snapshot. + pub fn create_session( + bindings: &Decider, + store: &mut wasmtime::Store, + snapshot: Option<&[u8]>, + ) -> wasmtime::Result { + bindings.interface0.session().call_constructor(store, snapshot) + } - pub fn snapshot( - bindings: &Decider, - store: &mut wasmtime::Store, - session: Session, - ) -> wasmtime::Result>> { - bindings.interface0.session().call_snapshot(store, session) - } + /// Calls the session's `evolve()` method to apply one stored event. + pub fn evolve( + bindings: &Decider, + store: &mut wasmtime::Store, + session: Session, + events: &[AnyEnvelope], + ) -> wasmtime::Result> { + bindings.interface0.session().call_evolve(store, session, events) + } + + /// Calls the session's `decide()` method to evaluate a command. + pub fn decide( + bindings: &Decider, + store: &mut wasmtime::Store, + session: Session, + command: &CommandEnvelope, + ) -> wasmtime::Result, DecideError>> { + bindings.interface0.session().call_decide(store, session, command) + } + + /// Calls the session's `snapshot()` method to serialize its current state. + pub fn snapshot( + bindings: &Decider, + store: &mut wasmtime::Store, + session: Session, + ) -> wasmtime::Result>> { + bindings.interface0.session().call_snapshot(store, session) + } - pub fn drop_session( - _bindings: &Decider, - store: &mut wasmtime::Store, - session: Session, - ) -> wasmtime::Result<()> { - session.resource_drop(store) + /// Drops the host's handle to a guest `session` resource, releasing it in the guest. + pub fn drop_session( + _bindings: &Decider, + store: &mut wasmtime::Store, + session: Session, + ) -> wasmtime::Result<()> { + session.resource_drop(store) + } } + + pub use bindings::exports::trogon::decider::handler::{ + AnyEnvelope, CommandEnvelope, CommandSpec, DecideError, DomainError, Guest, GuestSession, ModuleDescriptor, + WritePrecondition, + }; + pub use bindings::{ + Decider, DeciderPre, call_descriptor, call_stream_id, create_session, decide, drop_session, evolve, snapshot, + }; } diff --git a/rsworkspace/crates/trogon-decider-wit/wit/world.wit b/rsworkspace/crates/trogon-decider-wit/wit/world.wit index a40929117..166a32f0d 100644 --- a/rsworkspace/crates/trogon-decider-wit/wit/world.wit +++ b/rsworkspace/crates/trogon-decider-wit/wit/world.wit @@ -1,37 +1,94 @@ -package trogon:decider@0.1.0; +// 0.2.0: `domain-error` gained `details`, changing the record's ABI layout. Guest and host +// bindings must be regenerated from the same WIT version; this is not wire-compatible with 0.1.0. +package trogon:decider@0.2.0; +/// Contract every compiled Trogon decider component exports. interface handler { - record any-envelope { %type: string, payload: list } - record command-envelope { %type: string, payload: list } - record domain-error { code: string, message: string } + /// A stored event envelope: its persistent type name plus the encoded domain event payload. + record any-envelope { + /// Persistent event type name stored alongside the payload. + %type: string, + /// Encoded domain event payload. + payload: list, + } + + /// An incoming command envelope: its command type plus the encoded command payload. + record command-envelope { + /// Persistent command type name stored alongside the payload. + %type: string, + /// Encoded domain command payload. + payload: list, + } + + /// A structured guest-side error surfaced to the host. + record domain-error { + /// Stable, machine-readable rejection code (see `Decider::decide-error-code` on the guest). + code: string, + /// Human-readable description of the failure. + message: string, + /// Ordered key/value pairs carrying the error's source chain, most specific cause last. + details: list>, + } + + /// Optimistic concurrency precondition a command's emitted events must satisfy on append. + variant write-precondition { + /// Append without checking the stream's current position. + any, + /// Append only if the stream already has at least one event. + stream-exists, + /// Append only if the stream has no events. + no-stream, + } - variant write-precondition { any, stream-exists, no-stream } + /// One command type a decider module handles, plus the write precondition it requires. record command-spec { + /// Persistent command type name this spec covers. command-type: string, + /// Write precondition required for this command's events, or none for no constraint. write-precondition: option, } + + /// Static identity and command coverage of a compiled decider module, returned by `descriptor`. record module-descriptor { + /// Module name. Combined with `version`, identifies the module for snapshot routing. name: string, + /// Module version. Opaque to the runtime; used only for identity, not ordering. version: string, + /// Command types this module handles. commands: list, } + /// Outcome of a failed `session.decide` call. variant decide-error { + /// The command was understood but rejected by domain rules. rejected(domain-error), + /// The guest could not evaluate the command (e.g. a decode failure). faulted(domain-error), } + /// Returns this module's static identity and the commands it handles. descriptor: func() -> module-descriptor; + + /// Resolves the stream identifier a command targets, without evaluating it. stream-id: func(command: command-envelope) -> result; + /// One command's execution lifecycle: replay prior events, decide, then snapshot. resource session { + /// Creates a session, optionally resuming from a previously taken snapshot. constructor(snapshot: option>); + + /// Applies one stored event to the session's in-guest state. evolve: func(events: list) -> result<_, domain-error>; + + /// Evaluates a command against the current state, returning the events it produces. decide: func(command: command-envelope) -> result, decide-error>; + + /// Serializes the session's current state, or `none` if the module does not snapshot. snapshot: func() -> option>; } } +/// The world every compiled Trogon decider component implements. world decider { export handler; } diff --git a/rsworkspace/crates/trogon-decider/Cargo.toml b/rsworkspace/crates/trogon-decider/Cargo.toml index 221f0d875..465d714e6 100644 --- a/rsworkspace/crates/trogon-decider/Cargo.toml +++ b/rsworkspace/crates/trogon-decider/Cargo.toml @@ -10,6 +10,7 @@ repository = "https://github.com/TrogonStack/trogonai" test-support = [] [lints.rust] +missing_docs = "warn" warnings = "deny" unexpected_cfgs = { level = "deny", check-cfg = ['cfg(coverage)', 'cfg(dylint_lib, values(any()))'] } diff --git a/rsworkspace/crates/trogon-decider/README.md b/rsworkspace/crates/trogon-decider/README.md new file mode 100644 index 000000000..6f93b8255 --- /dev/null +++ b/rsworkspace/crates/trogon-decider/README.md @@ -0,0 +1,18 @@ +# trogon-decider + +The `trogon-*` prefix (per [ADR#0002](../../../docs/adr/0002-rust-crate-boundaries.md)) marks +first-party platform crates whose public contract does not depend on this repository's own +product assumptions; this crate is the pure decision-logic domain: the `Decider` trait +(`stream_id`, `initial_state`, `evolve`, `decide`), the `Decision` enum +(`Decision::Events` and the `Decision::act()` multi-step builder), `WritePrecondition`, and the +`EventEncode`/`EventType`/`EventDecode` codec traits events must implement to be replayed and +persisted. It knows nothing about storage, transport, or WASM; those live in +`trogon-decider-runtime`, `trogon-decider-nats`, and `trogon-decider-wasm-runtime` respectively. + +It also ships `testing` (feature `test-support`): `TestCase`, a given/when/then typestate +builder for asserting a `Decider`'s behavior directly against a `History` of prior events, +without any storage adapter. + +See the crate's own doc comment (`src/lib.rs`) for a complete `Decider` implementation, and +[Decider Platform](../../../docs/architecture/decider.md) for how this trait's output flows +through the native and WASM execution paths. diff --git a/rsworkspace/crates/trogon-decider/src/event/codec/event_payload_error.rs b/rsworkspace/crates/trogon-decider/src/event/codec/event_payload_error.rs index 5679d8f50..3b1310aef 100644 --- a/rsworkspace/crates/trogon-decider/src/event/codec/event_payload_error.rs +++ b/rsworkspace/crates/trogon-decider/src/event/codec/event_payload_error.rs @@ -14,7 +14,10 @@ pub enum EventPayloadError { MissingEvent, /// The envelope event type is not recognized by this payload decoder. #[error("unknown event type '{event_type}'")] - UnknownEventType { event_type: String }, + UnknownEventType { + /// The unrecognized event type stored with the envelope. + event_type: String, + }, } impl EventPayloadError { diff --git a/rsworkspace/crates/trogon-decider/src/lib.rs b/rsworkspace/crates/trogon-decider/src/lib.rs index fa415cb64..8d89ecd73 100644 --- a/rsworkspace/crates/trogon-decider/src/lib.rs +++ b/rsworkspace/crates/trogon-decider/src/lib.rs @@ -150,19 +150,6 @@ pub trait Decider: Sized { /// Decides which events the command produces against the current state. fn decide(state: &Self::State, command: &Self) -> Result, Self::DecideError>; - - /// Classifies a [`decide`](Self::decide) rejection into a stable, - /// machine-readable code. - /// - /// The WASM bridge surfaces this as the `code` field of the WIT - /// `domain-error`, so hosts and conformance suites can distinguish - /// rejection reasons (e.g. `already-exists` vs `deleted`) without parsing - /// the human-readable message. Defaults to `"rejected"`; override to map - /// each [`DecideError`](Self::DecideError) variant to its own code. - fn decide_error_code(error: &Self::DecideError) -> &str { - let _ = error; - "rejected" - } } #[cfg(test)] diff --git a/rsworkspace/crates/trogon-decider/src/testing.rs b/rsworkspace/crates/trogon-decider/src/testing.rs index 41e5c665c..67fa613c2 100644 --- a/rsworkspace/crates/trogon-decider/src/testing.rs +++ b/rsworkspace/crates/trogon-decider/src/testing.rs @@ -231,17 +231,19 @@ impl Default for History { /// A completed event expectation. /// -/// The value records the stream id, prior history, and events emitted by one -/// completed [`TestCase`]. Feed [`history`](Self::history) into the next case when a -/// test needs to model multiple commands against the same stream. +/// The value records the stream id, prior history, events emitted by one completed +/// [`TestCase`], and the state produced by replaying those events. Feed +/// [`history`](Self::history) into the next case when a test needs to model multiple +/// commands against the same stream. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ThenEvents { +pub struct ThenEvents { stream_id: String, given: History, then: Events, + resulting_state: State, } -impl ThenEvents { +impl ThenEvents { /// Stream id resolved from the command used in the test case. pub fn stream_id(&self) -> &str { &self.stream_id @@ -256,9 +258,19 @@ impl ThenEvents { pub fn then_events(&self) -> &[Event] { self.then.as_slice() } + + /// State produced by replaying [`then_events`](Self::then_events) on top of the + /// state the command decided against. + /// + /// This is the post-decision state: `decide` followed by `evolve` over every + /// emitted event. Use [`then_state`](Self::then_state) to assert it inline, or + /// this accessor to inspect it directly. + pub fn resulting_state(&self) -> &State { + &self.resulting_state + } } -impl ThenEvents +impl ThenEvents where Event: Clone, { @@ -270,9 +282,82 @@ where } } +impl ThenEvents +where + State: PartialEq + Debug, +{ + /// Asserts the state produced after this command's events are replayed. + /// + /// # Example + /// + /// ``` + /// # use trogon_decider::{Decider, Decision}; + /// # use trogon_decider::testing::TestCase; + /// # + /// # #[derive(Debug, Clone, PartialEq, Eq)] + /// # enum AccountState { + /// # Missing, + /// # Open, + /// # } + /// # + /// # #[derive(Debug, Clone, PartialEq, Eq)] + /// # enum AccountEvent { + /// # Opened, + /// # } + /// # + /// # #[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] + /// # enum AccountError { + /// # #[error("{self:?}")] + /// # AlreadyOpen, + /// # } + /// # + /// # #[derive(Debug, Clone, PartialEq, Eq)] + /// # struct OpenAccount { + /// # account_id: &'static str, + /// # } + /// # + /// # impl Decider for OpenAccount { + /// # type StreamId = str; + /// # type State = AccountState; + /// # type Event = AccountEvent; + /// # type DecideError = AccountError; + /// # type EvolveError = AccountError; + /// # + /// # fn stream_id(&self) -> &Self::StreamId { self.account_id } + /// # fn initial_state() -> Self::State { AccountState::Missing } + /// # fn evolve(_state: Self::State, event: &Self::Event) -> Result { + /// # match event { + /// # AccountEvent::Opened => Ok(AccountState::Open), + /// # } + /// # } + /// # fn decide(state: &Self::State, _command: &Self) -> Result, Self::DecideError> { + /// # match state { + /// # AccountState::Missing => Ok(Decision::event(AccountEvent::Opened)), + /// # AccountState::Open => Err(AccountError::AlreadyOpen), + /// # } + /// # } + /// # } + /// # + /// let opened = TestCase::::new() + /// .given_no_history() + /// .when(OpenAccount { account_id: "account-1" }) + /// .then([AccountEvent::Opened]) + /// .then_state(AccountState::Open); + /// + /// assert_eq!(opened.resulting_state(), &AccountState::Open); + /// ``` + pub fn then_state(self, expected: State) -> Self { + assert_eq!( + self.resulting_state, expected, + "then_state(...) resulting state did not match expectation" + ); + self + } +} + /// A completed error expectation. /// -/// The value records the stream id, prior history, and decision error from one +/// The value records the stream id, prior history, and decision error of one /// completed [`TestCase`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ThenError { @@ -290,7 +375,7 @@ pub enum DecisionEvaluationFailure { #[doc(hidden)] pub type DecisionEvaluation = Result< - Events<::Event>, + (Events<::Event>, ::State), DecisionEvaluationFailure<::DecideError, ::EvolveError>, >; @@ -582,7 +667,7 @@ where "then_error(...) expected a decide error, but emitted events could not be evolved:\nexpected error = {:?}\nactual evolve error = {:?}", expected_error, error, ), - Ok(events) => panic!( + Ok((events, _state)) => panic!( "then_error(...) expected an error, but decide(...) emitted events:\nexpected error = {:?}\nactual events = {:?}", expected_error, events.as_slice(), @@ -612,8 +697,10 @@ where C: Decider, C::StreamId: std::fmt::Display, { + /// Value produced once the expectation has been asserted against a decision. type Output; + /// Asserts `actual` matches this expectation, panicking with a diagnostic otherwise. fn assert_matches(self, given: History, command: &C, actual: DecisionEvaluation) -> Self::Output; } @@ -626,7 +713,7 @@ where C::StreamId: std::fmt::Display, E: Into, { - type Output = ThenEvents; + type Output = ThenEvents; fn assert_matches(self, given: History, command: &C, actual: DecisionEvaluation) -> Self::Output { assert_events_match::( @@ -646,7 +733,7 @@ where C::EvolveError: Debug, C::StreamId: std::fmt::Display, { - type Output = ThenEvents; + type Output = ThenEvents; fn assert_matches(self, given: History, command: &C, actual: DecisionEvaluation) -> Self::Output { assert!( @@ -666,7 +753,7 @@ where C::StreamId: std::fmt::Display, E: Into, { - type Output = ThenEvents; + type Output = ThenEvents; fn assert_matches(self, given: History, command: &C, actual: DecisionEvaluation) -> Self::Output { assert!( @@ -687,7 +774,7 @@ fn assert_events_match( given: History, command: &C, actual: DecisionEvaluation, -) -> ThenEvents +) -> ThenEvents where C: Decider, C::Event: Clone + PartialEq + Debug, @@ -696,7 +783,7 @@ where C::StreamId: std::fmt::Display, { match actual { - Ok(events) => { + Ok((events, resulting_state)) => { assert_eq!( events.as_slice(), expected.as_slice(), @@ -706,6 +793,7 @@ where stream_id: command.stream_id().to_string(), given, then: events, + resulting_state, } } Err(DecisionEvaluationFailure::Decide(error)) => panic!( @@ -723,11 +811,11 @@ fn decide_events(state: C::State, command: &C) -> DecisionEvaluation where C: Decider, { - let (_, events) = evaluate_decision(state, command).map_err(|failure| match failure { + let (state, events) = evaluate_decision(state, command).map_err(|failure| match failure { DecisionFailure::Decide(error) => DecisionEvaluationFailure::Decide(error), DecisionFailure::Evolve(error) => DecisionEvaluationFailure::Evolve(error), })?; - Ok(events) + Ok((events, state)) } fn assert_expected_state(expected: Option>, actual: &State) { @@ -736,7 +824,10 @@ fn assert_expected_state(expected: Option>, actual: } } +mod codec; mod private; +pub use codec::assert_round_trips; + #[cfg(test)] mod tests; diff --git a/rsworkspace/crates/trogon-decider/src/testing/codec.rs b/rsworkspace/crates/trogon-decider/src/testing/codec.rs new file mode 100644 index 000000000..235ba2b09 --- /dev/null +++ b/rsworkspace/crates/trogon-decider/src/testing/codec.rs @@ -0,0 +1,96 @@ +use std::fmt::Debug; + +use crate::{EventData, EventDecode, EventDecodeOutcome, EventEncode, EventType}; + +/// Asserts that an event survives a codec round trip: [`EventEncode::encode`] followed +/// by [`EventDecode::decode`] against the event's own [`EventType::event_type`]. +/// +/// This exercises the same boundary the storage adapter crosses at runtime, so it +/// catches codec bugs a pure `evolve`/`decide` test cannot: a decoder that does not +/// recognize its own encoder's event type, a payload that does not survive +/// serialization, or a decoder that resolves to [`EventDecodeOutcome::Skipped`] +/// instead of [`EventDecodeOutcome::Decoded`]. +/// +/// It does not exercise aliases, migrations, or upcasters on their own: those map a +/// *historical* event type string to the current event, so assert them directly by +/// building [`EventData`] with the historical type and calling +/// [`EventDecode::decode`]. +/// +/// # Example +/// +/// ``` +/// use trogon_decider::testing::assert_round_trips; +/// use trogon_decider::{EventData, EventDecode, EventDecodeOutcome, EventEncode, EventType}; +/// +/// #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +/// #[error("widget event codec failure")] +/// struct WidgetEventCodecError; +/// +/// #[derive(Debug, Clone, PartialEq, Eq)] +/// enum WidgetEvent { +/// Created { name: String }, +/// } +/// +/// impl EventType for WidgetEvent { +/// type Error = WidgetEventCodecError; +/// +/// fn event_type(&self) -> Result<&'static str, Self::Error> { +/// match self { +/// Self::Created { .. } => Ok("widget.created.v2"), +/// } +/// } +/// } +/// +/// impl EventEncode for WidgetEvent { +/// type Error = WidgetEventCodecError; +/// +/// fn encode(&self) -> Result, Self::Error> { +/// match self { +/// Self::Created { name } => Ok(name.clone().into_bytes()), +/// } +/// } +/// } +/// +/// impl EventDecode for WidgetEvent { +/// type Error = WidgetEventCodecError; +/// +/// fn decode(event: EventData<'_>) -> Result, Self::Error> { +/// let name = String::from_utf8(event.payload.to_vec()).map_err(|_| WidgetEventCodecError)?; +/// Ok(match event.event_type { +/// // Current wire name. +/// "widget.created.v2" => EventDecodeOutcome::Decoded(Self::Created { name }), +/// // Historical alias kept so old envelopes still replay. +/// "widget.created" => EventDecodeOutcome::Decoded(Self::Created { name }), +/// _ => EventDecodeOutcome::Skipped, +/// }) +/// } +/// } +/// +/// assert_round_trips(WidgetEvent::Created { name: "widget-1".to_string() }); +/// +/// // Aliases are asserted separately, against the historical event type string. +/// let aliased = WidgetEvent::decode(EventData::new("widget.created", b"widget-1")).unwrap(); +/// assert_eq!(aliased, EventDecodeOutcome::Decoded(WidgetEvent::Created { name: "widget-1".to_string() })); +/// ``` +pub fn assert_round_trips(event: E) +where + E: EventType + EventEncode + EventDecode + PartialEq + Debug, +{ + let event_type = event + .event_type() + .unwrap_or_else(|error| panic!("assert_round_trips(...) failed to resolve event type: {error}")); + let payload = event + .encode() + .unwrap_or_else(|error| panic!("assert_round_trips(...) failed to encode event: {error}")); + let decoded = E::decode(EventData::new(event_type, &payload)) + .unwrap_or_else(|error| panic!("assert_round_trips(...) failed to decode event: {error}")); + + assert_eq!( + decoded, + EventDecodeOutcome::Decoded(event), + "assert_round_trips(...) decoded event did not match the original after a codec round trip" + ); +} + +#[cfg(test)] +mod tests; diff --git a/rsworkspace/crates/trogon-decider/src/testing/codec/tests.rs b/rsworkspace/crates/trogon-decider/src/testing/codec/tests.rs new file mode 100644 index 000000000..0dbb2c7cf --- /dev/null +++ b/rsworkspace/crates/trogon-decider/src/testing/codec/tests.rs @@ -0,0 +1,113 @@ +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("widget event codec failure")] +struct WidgetEventCodecError; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum WidgetEvent { + Created { name: String }, + Renamed { name: String }, +} + +impl EventType for WidgetEvent { + type Error = WidgetEventCodecError; + + fn event_type(&self) -> Result<&'static str, Self::Error> { + match self { + Self::Created { .. } => Ok("widget.created.v2"), + Self::Renamed { .. } => Ok("widget.renamed"), + } + } +} + +impl EventEncode for WidgetEvent { + type Error = WidgetEventCodecError; + + fn encode(&self) -> Result, Self::Error> { + match self { + Self::Created { name } | Self::Renamed { name } => Ok(name.clone().into_bytes()), + } + } +} + +impl EventDecode for WidgetEvent { + type Error = WidgetEventCodecError; + + fn decode(event: EventData<'_>) -> Result, Self::Error> { + let name = String::from_utf8(event.payload.to_vec()).map_err(|_| WidgetEventCodecError)?; + Ok(match event.event_type { + "widget.created.v2" => EventDecodeOutcome::Decoded(Self::Created { name }), + // Historical alias kept so envelopes written before the v2 rename still replay. + "widget.created" => EventDecodeOutcome::Decoded(Self::Created { name }), + "widget.renamed" => EventDecodeOutcome::Decoded(Self::Renamed { name }), + _ => EventDecodeOutcome::Skipped, + }) + } +} + +#[test] +fn assert_round_trips_passes_for_current_event_type() { + assert_round_trips(WidgetEvent::Created { + name: "widget-1".to_string(), + }); + assert_round_trips(WidgetEvent::Renamed { + name: "widget-2".to_string(), + }); +} + +#[test] +fn aliased_event_type_decodes_to_the_current_variant() { + let decoded = WidgetEvent::decode(EventData::new("widget.created", b"widget-1")).unwrap(); + + assert_eq!( + decoded, + EventDecodeOutcome::Decoded(WidgetEvent::Created { + name: "widget-1".to_string(), + }) + ); +} + +#[test] +fn unrecognized_event_type_is_skipped_not_errored() { + let decoded = WidgetEvent::decode(EventData::new("other.event", b"payload")).unwrap(); + + assert_eq!(decoded, EventDecodeOutcome::Skipped); +} + +#[test] +fn assert_round_trips_panics_when_decoder_skips_the_event_type() { + #[derive(Debug, Clone, PartialEq, Eq)] + struct AlwaysSkips; + + impl EventType for AlwaysSkips { + type Error = WidgetEventCodecError; + + fn event_type(&self) -> Result<&'static str, Self::Error> { + Ok("always.skips") + } + } + + impl EventEncode for AlwaysSkips { + type Error = WidgetEventCodecError; + + fn encode(&self) -> Result, Self::Error> { + Ok(Vec::new()) + } + } + + impl EventDecode for AlwaysSkips { + type Error = WidgetEventCodecError; + + fn decode(_event: EventData<'_>) -> Result, Self::Error> { + Ok(EventDecodeOutcome::Skipped) + } + } + + let panic = catch_unwind(AssertUnwindSafe(|| assert_round_trips(AlwaysSkips))).unwrap_err(); + let message = panic.downcast_ref::().cloned().unwrap_or_default(); + + assert!(message.contains("did not match the original after a codec round trip")); +} diff --git a/rsworkspace/crates/trogon-decider/src/testing/tests.rs b/rsworkspace/crates/trogon-decider/src/testing/tests.rs index 1e5438e41..18948950b 100644 --- a/rsworkspace/crates/trogon-decider/src/testing/tests.rs +++ b/rsworkspace/crates/trogon-decider/src/testing/tests.rs @@ -534,6 +534,31 @@ fn error_mismatch_shows_expected_vs_actual() { assert!(message.contains("Registered")); } +#[test] +fn then_state_exposes_and_pins_the_resulting_state() { + let registered = TestCase::::new() + .given_no_history() + .when(TestCommand::register("alpha")) + .then([TestEvent::Registered { + id: "alpha".to_string(), + }]) + .then_state(TestState::Present { enabled: true }); + + assert_eq!(registered.resulting_state(), &TestState::Present { enabled: true }); +} + +#[test] +#[should_panic(expected = "then_state(...) resulting state did not match expectation")] +fn then_state_mismatch_panics() { + TestCase::::new() + .given_no_history() + .when(TestCommand::register("alpha")) + .then([TestEvent::Registered { + id: "alpha".to_string(), + }]) + .then_state(TestState::Missing); +} + #[test] fn panic_message_returns_empty_for_unknown_payload() { let panic = catch_unwind(AssertUnwindSafe(|| std::panic::panic_any(123_u8))).unwrap_err(); diff --git a/rsworkspace/crates/trogon-decider/src/tests.rs b/rsworkspace/crates/trogon-decider/src/tests.rs index f60741e43..5cd22b307 100644 --- a/rsworkspace/crates/trogon-decider/src/tests.rs +++ b/rsworkspace/crates/trogon-decider/src/tests.rs @@ -101,11 +101,6 @@ fn act_propagates_evolve_errors() { ); } -#[test] -fn decide_error_code_defaults_to_rejected() { - assert_eq!(TestCommand::decide_error_code(&TestError::Decide), "rejected"); -} - #[test] fn decide_exposes_typed_stream_id() { let command = TestCommand; diff --git a/rsworkspace/crates/trogon-decider/tests/trybuild.rs b/rsworkspace/crates/trogon-decider/tests/trybuild.rs index 206abc35c..75bf5b8f2 100644 --- a/rsworkspace/crates/trogon-decider/tests/trybuild.rs +++ b/rsworkspace/crates/trogon-decider/tests/trybuild.rs @@ -1,3 +1,4 @@ +//! Compile-fail and compile-pass cases for `TestCase`'s given/when/then typestate chain. #![allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)] #[test] diff --git a/rsworkspace/crates/trogon-decider/tests/ui/fail/after_then.stderr b/rsworkspace/crates/trogon-decider/tests/ui/fail/after_then.stderr index 7c93b5f1a..c7af1d213 100644 --- a/rsworkspace/crates/trogon-decider/tests/ui/fail/after_then.stderr +++ b/rsworkspace/crates/trogon-decider/tests/ui/fail/after_then.stderr @@ -1,4 +1,4 @@ -error[E0599]: no method named `when` found for struct `ThenEvents` in the current scope +error[E0599]: no method named `when` found for struct `ThenEvents` in the current scope --> tests/ui/fail/after_then.rs:13:10 | 9 | TestCase::::new() @@ -12,6 +12,6 @@ error[E0599]: no method named `when` found for struct `ThenEvents` in the | | ----------------- method `when` is available on `TestCase>` 12 | | .then([TestEvent::Registered]) 13 | | .when(TestCommand); - | | -^^^^ method not found in `ThenEvents` + | | -^^^^ method not found in `ThenEvents` | |_________| | diff --git a/rsworkspace/crates/trogon-semconv/src/gen/attribute.rs b/rsworkspace/crates/trogon-semconv/src/gen/attribute.rs index 774a211d3..dd49283c5 100644 --- a/rsworkspace/crates/trogon-semconv/src/gen/attribute.rs +++ b/rsworkspace/crates/trogon-semconv/src/gen/attribute.rs @@ -165,12 +165,42 @@ pub const ACTION: &str = "action"; /// Authentication mode used for the NATS connection pub const AUTH: &str = "auth"; +/// Command type identifier for the decider command being processed +pub const COMMAND_TYPE: &str = "command_type"; + /// Config option id being set on an ACP session pub const CONFIG_ID: &str = "config_id"; /// Working directory for a new ACP session pub const CWD: &str = "cwd"; +/// Outcome of a command execution attempt +pub const DECISION_OUTCOME: &str = "decision_outcome"; + +/// Allowed values for the `decision_outcome` attribute +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DecisionOutcome { + /// The command was decided and its events appended + Decided, + /// The command was rejected by domain decision logic + Rejected, + /// Command execution faulted on an infrastructure or guest execution failure + Faulted, +} + +impl DecisionOutcome { + /// Returns the wire value recorded for this attribute member. + #[must_use] + pub const fn as_str(&self) -> &'static str { + match self { + Self::Decided => "decided", + Self::Rejected => "rejected", + Self::Faulted => "faulted", + } + } +} + /// GitHub webhook delivery identifier pub const DELIVERY: &str = "delivery"; @@ -186,6 +216,36 @@ pub const EVENT_TYPE: &str = "event_type"; /// Extension method name on the acp.client.ext span pub const EXT_METHOD: &str = "ext_method"; +/// Phase of WASM guest command execution +pub const GUEST_PHASE: &str = "guest_phase"; + +/// Allowed values for the `guest_phase` attribute +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum GuestPhase { + /// Instantiating the WASM guest component + Instantiate, + /// Replaying prior events into the guest session + Replay, + /// Calling the guest's decide export + Decide, + /// Reading or writing the guest session's snapshot + Snapshot, +} + +impl GuestPhase { + /// Returns the wire value recorded for this attribute member. + #[must_use] + pub const fn as_str(&self) -> &'static str { + match self { + Self::Instantiate => "instantiate", + Self::Replay => "replay", + Self::Decide => "decide", + Self::Snapshot => "snapshot", + } + } +} + /// MCP servers configured for a new ACP session pub const MCP_SERVERS: &str = "mcp_servers"; @@ -261,6 +321,12 @@ pub const METHOD_ID: &str = "method_id"; /// Mode id being set on an ACP session pub const MODE_ID: &str = "mode_id"; +/// Name of the WASM decider module handling the command +pub const MODULE_NAME: &str = "module_name"; + +/// Version of the WASM decider module handling the command +pub const MODULE_VERSION: &str = "module_version"; + /// Session id of the newly forked ACP session pub const NEW_SESSION_ID: &str = "new_session_id"; @@ -438,6 +504,42 @@ pub const SERVERS: &str = "servers"; /// ACP session identifier; set on many session-scoped spans pub const SESSION_ID: &str = "session_id"; +/// Outcome of the snapshot read at the start of command execution +pub const SNAPSHOT_OUTCOME: &str = "snapshot_outcome"; + +/// Allowed values for the `snapshot_outcome` attribute +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SnapshotOutcome { + /// A snapshot was found and used as the replay starting point + Hit, + /// No snapshot existed; state was rebuilt from the initial state + Miss, + /// A snapshot read failure was discarded and replay restarted from the initial state + DiscardedReadFailure, + /// A snapshot ahead of the stream's current position was discarded + DiscardedAheadOfStream, + /// A snapshot problem was detected and the command failed instead of discarding the snapshot + Failed, +} + +impl SnapshotOutcome { + /// Returns the wire value recorded for this attribute member. + #[must_use] + pub const fn as_str(&self) -> &'static str { + match self { + Self::Hit => "hit", + Self::Miss => "miss", + Self::DiscardedReadFailure => "discarded_read_failure", + Self::DiscardedAheadOfStream => "discarded_ahead_of_stream", + Self::Failed => "failed", + } + } +} + +/// Whether a decider snapshot write succeeded +pub const SNAPSHOT_WRITE_SUCCESS: &str = "snapshot_write_success"; + /// Identifier of the event stream the schedule record belongs to pub const STREAM_ID: &str = "stream_id"; @@ -456,6 +558,30 @@ pub const SUCCESS: &str = "success"; /// Connection timeout in seconds pub const TIMEOUT_SECS: &str = "timeout_secs"; +/// Classification of a WASM guest call trap +pub const TRAP_CLASSIFICATION: &str = "trap_classification"; + +/// Allowed values for the `trap_classification` attribute +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum TrapClassification { + /// The guest call exceeded its wall-clock epoch deadline + DeadlineExceeded, + /// The guest call trapped for a reason other than the epoch deadline, e.g. fuel exhaustion, a memory limit, or an ABI failure + Trap, +} + +impl TrapClassification { + /// Returns the wire value recorded for this attribute member. + #[must_use] + pub const fn as_str(&self) -> &'static str { + match self { + Self::DeadlineExceeded => "deadline_exceeded", + Self::Trap => "trap", + } + } +} + /// Telegram update identifier pub const UPDATE_ID: &str = "update_id"; @@ -468,6 +594,36 @@ pub const WEBHOOK_ID: &str = "webhook_id"; /// GitLab webhook UUID pub const WEBHOOK_UUID: &str = "webhook_uuid"; +/// Optimistic concurrency precondition applied to a stream append +pub const WRITE_PRECONDITION: &str = "write_precondition"; + +/// Allowed values for the `write_precondition` attribute +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum WritePrecondition { + /// Append regardless of the stream's current position + Any, + /// Append only if the stream already has at least one event + StreamExists, + /// Append only if the stream has no events + NoStream, + /// Append only if the stream is still at the observed position + At, +} + +impl WritePrecondition { + /// Returns the wire value recorded for this attribute member. + #[must_use] + pub const fn as_str(&self) -> &'static str { + match self { + Self::Any => "any", + Self::StreamExists => "stream_exists", + Self::NoStream => "no_stream", + Self::At => "at", + } + } +} + /// Standard OTel server.address. Emitted by http.server.request pub const SERVER_ADDRESS: &str = "server.address"; diff --git a/rsworkspace/crates/trogon-semconv/src/gen/metric.rs b/rsworkspace/crates/trogon-semconv/src/gen/metric.rs index aa565a68f..bb1e33dd3 100644 --- a/rsworkspace/crates/trogon-semconv/src/gen/metric.rs +++ b/rsworkspace/crates/trogon-semconv/src/gen/metric.rs @@ -56,6 +56,206 @@ pub fn build_acp_requests(meter: &opentelemetry::metrics::Meter) -> opentelemetr .build() } +/// Optimistic concurrency conflicts (wrong expected stream version) surfaced by an append +/// +/// Instrument: `counter`. Unit: `{conflict}`. +pub const DECIDER_APPEND_CONFLICTS: &str = "decider.append.conflicts"; + +/// Optimistic concurrency conflicts (wrong expected stream version) surfaced by an append +/// +/// Instrument: `counter`. Unit: `{conflict}`. +pub fn build_decider_append_conflicts(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_APPEND_CONFLICTS) + .with_description("Optimistic concurrency conflicts (wrong expected stream version) surfaced by an append.") + .with_unit("{conflict}") + .build() +} + +/// Duration of appending decided events to a JetStream-backed stream +/// +/// Instrument: `histogram`. Unit: `s`. +pub const DECIDER_APPEND_DURATION: &str = "decider.append.duration"; + +/// Duration of appending decided events to a JetStream-backed stream +/// +/// Instrument: `histogram`. Unit: `s`. +pub fn build_decider_append_duration(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Histogram { + meter + .f64_histogram(DECIDER_APPEND_DURATION) + .with_description("Duration of appending decided events to a JetStream-backed stream.") + .with_unit("s") + .build() +} + +/// Duration of replaying events from an ordered, subject-filtered JetStream consumer +/// +/// Instrument: `histogram`. Unit: `s`. +pub const DECIDER_REPLAY_DURATION: &str = "decider.replay.duration"; + +/// Duration of replaying events from an ordered, subject-filtered JetStream consumer +/// +/// Instrument: `histogram`. Unit: `s`. +pub fn build_decider_replay_duration(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Histogram { + meter + .f64_histogram(DECIDER_REPLAY_DURATION) + .with_description("Duration of replaying events from an ordered, subject-filtered JetStream consumer.") + .with_unit("s") + .build() +} + +/// Events replayed to rebuild decider state before a command decision +/// +/// Instrument: `counter`. Unit: `{event}`. +pub const DECIDER_REPLAY_EVENTS: &str = "decider.replay.events"; + +/// Events replayed to rebuild decider state before a command decision +/// +/// Instrument: `counter`. Unit: `{event}`. +pub fn build_decider_replay_events(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_REPLAY_EVENTS) + .with_description("Events replayed to rebuild decider state before a command decision.") + .with_unit("{event}") + .build() +} + +/// Ordered-consumer replay attempts retried after a transient JetStream failure +/// +/// Instrument: `counter`. Unit: `{attempt}`. +pub const DECIDER_REPLAY_RETRIES: &str = "decider.replay.retries"; + +/// Ordered-consumer replay attempts retried after a transient JetStream failure +/// +/// Instrument: `counter`. Unit: `{attempt}`. +pub fn build_decider_replay_retries(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_REPLAY_RETRIES) + .with_description("Ordered-consumer replay attempts retried after a transient JetStream failure.") + .with_unit("{attempt}") + .build() +} + +/// JetStream KV snapshot read failures in the storage adapter +/// +/// Instrument: `counter`. Unit: `{failure}`. +pub const DECIDER_SNAPSHOT_KV_READ_FAILURES: &str = "decider.snapshot.kv_read_failures"; + +/// JetStream KV snapshot read failures in the storage adapter +/// +/// Instrument: `counter`. Unit: `{failure}`. +pub fn build_decider_snapshot_kv_read_failures( + meter: &opentelemetry::metrics::Meter, +) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_SNAPSHOT_KV_READ_FAILURES) + .with_description("JetStream KV snapshot read failures in the storage adapter.") + .with_unit("{failure}") + .build() +} + +/// JetStream KV snapshot write failures in the storage adapter +/// +/// Instrument: `counter`. Unit: `{failure}`. +pub const DECIDER_SNAPSHOT_KV_WRITE_FAILURES: &str = "decider.snapshot.kv_write_failures"; + +/// JetStream KV snapshot write failures in the storage adapter +/// +/// Instrument: `counter`. Unit: `{failure}`. +pub fn build_decider_snapshot_kv_write_failures( + meter: &opentelemetry::metrics::Meter, +) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_SNAPSHOT_KV_WRITE_FAILURES) + .with_description("JetStream KV snapshot write failures in the storage adapter.") + .with_unit("{failure}") + .build() +} + +/// Snapshot reads at the start of command execution, by outcome +/// +/// Instrument: `counter`. Unit: `{read}`. +pub const DECIDER_SNAPSHOT_READS: &str = "decider.snapshot.reads"; + +/// Snapshot reads at the start of command execution, by outcome +/// +/// Instrument: `counter`. Unit: `{read}`. +pub fn build_decider_snapshot_reads(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_SNAPSHOT_READS) + .with_description("Snapshot reads at the start of command execution, by outcome.") + .with_unit("{read}") + .build() +} + +/// Scheduled decider snapshot writes, by success +/// +/// Instrument: `counter`. Unit: `{write}`. +pub const DECIDER_SNAPSHOT_WRITES: &str = "decider.snapshot.writes"; + +/// Scheduled decider snapshot writes, by success +/// +/// Instrument: `counter`. Unit: `{write}`. +pub fn build_decider_snapshot_writes(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_SNAPSHOT_WRITES) + .with_description("Scheduled decider snapshot writes, by success.") + .with_unit("{write}") + .build() +} + +/// Duration of a WASM guest execution phase +/// +/// Instrument: `histogram`. Unit: `s`. +pub const DECIDER_WASM_EXECUTION_DURATION: &str = "decider.wasm.execution.duration"; + +/// Duration of a WASM guest execution phase +/// +/// Instrument: `histogram`. Unit: `s`. +pub fn build_decider_wasm_execution_duration( + meter: &opentelemetry::metrics::Meter, +) -> opentelemetry::metrics::Histogram { + meter + .f64_histogram(DECIDER_WASM_EXECUTION_DURATION) + .with_description("Duration of a WASM guest execution phase.") + .with_unit("s") + .build() +} + +/// Wasmtime fuel consumed by a WASM guest execution phase +/// +/// Instrument: `histogram`. Unit: `{fuel_unit}`. +pub const DECIDER_WASM_FUEL_CONSUMED: &str = "decider.wasm.fuel.consumed"; + +/// Wasmtime fuel consumed by a WASM guest execution phase +/// +/// Instrument: `histogram`. Unit: `{fuel_unit}`. +pub fn build_decider_wasm_fuel_consumed( + meter: &opentelemetry::metrics::Meter, +) -> opentelemetry::metrics::Histogram { + meter + .u64_histogram(DECIDER_WASM_FUEL_CONSUMED) + .with_description("Wasmtime fuel consumed by a WASM guest execution phase.") + .with_unit("{fuel_unit}") + .build() +} + +/// WASM guest call traps, by classification +/// +/// Instrument: `counter`. Unit: `{trap}`. +pub const DECIDER_WASM_TRAPS: &str = "decider.wasm.traps"; + +/// WASM guest call traps, by classification +/// +/// Instrument: `counter`. Unit: `{trap}`. +pub fn build_decider_wasm_traps(meter: &opentelemetry::metrics::Meter) -> opentelemetry::metrics::Counter { + meter + .u64_counter(DECIDER_WASM_TRAPS) + .with_description("WASM guest call traps, by classification.") + .with_unit("{trap}") + .build() +} + /// Execution schedule messages published /// /// Instrument: `counter`. Unit: `{message}`. diff --git a/rsworkspace/crates/trogon-semconv/src/gen/span.rs b/rsworkspace/crates/trogon-semconv/src/gen/span.rs index 7c46c20f9..1cca76169 100644 --- a/rsworkspace/crates/trogon-semconv/src/gen/span.rs +++ b/rsworkspace/crates/trogon-semconv/src/gen/span.rs @@ -138,6 +138,33 @@ pub const ACP_SESSION_SET_MODE: &str = "acp.session.set_mode"; /// Datadog webhook ingestion pub const DATADOG_WEBHOOK: &str = "datadog.webhook"; +/// Appending decided events to a JetStream-backed stream +pub const DECIDER_APPEND_STREAM: &str = "decider.append_stream"; + +/// Execution of a single decider command, from snapshot and stream read through decide and append +pub const DECIDER_EXECUTE_COMMAND: &str = "decider.execute_command"; + +/// Reading a decider snapshot before replaying a command +pub const DECIDER_READ_SNAPSHOT: &str = "decider.read_snapshot"; + +/// Replaying events from an ordered, subject-filtered JetStream consumer +pub const DECIDER_REPLAY_STREAM: &str = "decider.replay_stream"; + +/// Calling a WASM decider guest's decide export +pub const DECIDER_WASM_DECIDE: &str = "decider.wasm.decide"; + +/// Instantiating a WASM decider guest component for a command +pub const DECIDER_WASM_INSTANTIATE: &str = "decider.wasm.instantiate"; + +/// Replaying prior events into a WASM decider guest session +pub const DECIDER_WASM_REPLAY: &str = "decider.wasm.replay"; + +/// Reading or writing a WASM decider guest session's snapshot +pub const DECIDER_WASM_SNAPSHOT: &str = "decider.wasm.snapshot"; + +/// Writing a decider snapshot after a command decision +pub const DECIDER_WRITE_SNAPSHOT: &str = "decider.write_snapshot"; + /// ACP client method dispatch. Span name is `dispatch_client_method` (no namespace) today pub const DISPATCH_CLIENT_METHOD: &str = "dispatch_client_method";