diff --git a/.gitignore b/.gitignore index d5f18343..d8a90346 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ target *.nix.orig *.nix.rej /.direnv +/.ofborg-data diff --git a/Architecture.md b/Architecture.md new file mode 100644 index 00000000..cb10abbc --- /dev/null +++ b/Architecture.md @@ -0,0 +1,175 @@ +# Architecture + +## Overview + +ofborg is a Rust-based CI system for [Nixpkgs](https://github.com/nixos/nixpkgs). It runs as a +[GitHub App](https://docs.github.com/en/apps) that automatically processes pull requests and issue +comments. The system uses **RabbitMQ** for asynchronous message passing between components. Each +component runs as an independent process communicating exclusively through exchanges and queues. + +## Components + +| Binary | Purpose | Consumes From | Publishes To | +|--------|---------|---------------|--------------| +| `github-webhook-receiver` | HTTP server receiving GitHub webhooks. Validates HMAC-SHA256 signatures, parses the event type, and routes raw JSON to RabbitMQ. | GitHub HTTP POST | `github-events` (topic exchange) | +| `github-comment-filter` | Consumes issue comment events, parses `@ofborg` commands (`build` / `eval`), checks ACL (repo authorization, trusted users), fetches PR data from GitHub API, and schedules build/eval jobs. | `build-inputs` queue | `build-inputs-{system}` queues (via default exchange), `mass-rebuild-check-jobs` (queue via default exchange), `build-results` (fanout exchange) | +| `evaluation-filter` | Filters pull request events for mass rebuild evaluation. Only processes Opened, Synchronize, Reopened, and Edited (base change) events on authorized repos. | `mass-rebuild-check-inputs` queue | `mass-rebuild-check-jobs` (queue via default exchange) | +| `mass-rebuilder` | Runs nixpkgs evaluation to determine which packages a PR impacts. Clones nixpkgs, merges the PR, runs nix-instantiate on release expressions, computes the diff of out-paths, and optionally publishes evaluation results to the hydra evaluator. | `mass-rebuild-check-jobs` queue | `hydra-eval-jobs` (queue via default exchange) | +| `github-comment-poster` | Creates and updates GitHub Check Runs on pull requests. On receipt of `QueuedBuildJobs` creates an in-progress check; on receipt of `BuildResult` creates a completed check run. | `build-results` queue (via `build-results` fanout) | GitHub Check Runs API | +| `stats` | Dual-purpose binary: (1) Prometheus metrics HTTP server at `/metrics`, (2) consumer of `EventMessage` payloads from the stats exchange for metric collection. | `stats-events` queue (via `stats` fanout) | HTTP Prometheus `/metrics` | + + +## Data Flow + +### 1. Webhook Reception + +GitHub sends webhook events to `github-webhook-receiver` via HTTP POST. The receiver validates the +`X-Hub-Signature-256` HMAC header, parses the `X-GitHub-Event` header, and publishes the raw JSON +body to the `github-events` **topic exchange** with a routing key of `{event}.{owner}/{repo}` (e.g., +`pull_request.nixos/nixpkgs`). + +``` +GitHub ──HTTP POST──▶ github-webhook-receiver ──publish──▶ github-events (topic) +``` + +### 2. Direct Build Flow (Issue Comments) + +When someone comments `@ofborg build` on a PR: + +1. **`github-comment-filter`** consumes the `issue_comment.*` event from `build-inputs` queue. +2. It parses the comment, checks the ACL (authorized repos, trusted users), fetches PR data from the + GitHub API, and publishes: + - A `BuildJob` message directly to `build-inputs-{system}` queues (one per architecture, via default exchange). + - A `QueuedBuildJobs` message to the `build-results` **fanout exchange** (for creating in-progress + check runs). + +``` +github-events ──issue_comment.*──▶ build-inputs ──▶ github-comment-filter + ├──▶ build-inputs-x86_64-linux + ├──▶ build-inputs-aarch64-linux + ├──▶ build-inputs-x86_64-darwin + ├──▶ build-inputs-aarch64-darwin + └──▶ build-results (fanout) +``` + +### 3. Mass Rebuild Flow (Pull Requests) + +When a PR is opened or updated: + +1. **`evaluation-filter`** consumes the `pull_request.*` event from `mass-rebuild-check-inputs`. + This queue has two bindings to `github-events`: `pull_request.*` (from the webhook-receiver) and + `pull_request.nixos/*` (from the evaluation-filter itself). +2. It filters for Opened, Synchronize, Reopened, and Edited (base change) actions on authorized + repos (`nixos/*`). +3. Publishes an `EvaluationJob` to `mass-rebuild-check-jobs`. + +``` +github-events ──pull_request.*──▶ mass-rebuild-check-inputs ──▶ evaluation-filter + └──▶ mass-rebuild-check-jobs +``` + +4. **`mass-rebuilder`** consumes `EvaluationJob` from `mass-rebuild-check-jobs`. It clones nixpkgs, + checks out the target branch, merges the PR, runs nix-instantiate on `nixos/release.nix` and + `pkgs/top-level/release.nix`, computes the diff of out-paths between base and merge, and + optionally publishes evaluation results to the `hydra-eval-jobs` queue. + +``` +mass-rebuild-check-jobs ──▶ mass-rebuilder ──▶ hydra-eval-jobs +``` + +### 4. Build Results + +**`github-comment-poster`** consumes from the `build-results` queue (bound to the `build-results` +fanout exchange). It receives two message types: +- `QueuedBuildJobs` — creates an in-progress GitHub Check Run on the PR. +- `BuildResult` — creates a completed GitHub Check Run (success/failure). + +``` +build-results (fanout) ──▶ build-results queue ──▶ github-comment-poster ──▶ GitHub Checks API +``` + +### 5. Stats / Metrics + +Components publish `EventMessage` payloads to the `stats` fanout exchange. The `stats` binary +consumes them from the `stats-events` queue and exposes collected metrics as Prometheus text format +at `/metrics` over HTTP. + +``` +components ──▶ stats (fanout) ──▶ stats-events queue ──▶ stats (Prometheus HTTP /metrics) +``` + +## RabbitMQ Topology + +### Exchanges + +| Exchange | Type | Purpose | +|----------|------|---------| +| `github-events` | Topic | Ingress for all raw GitHub webhook events. Routing keys: `{event}.{owner}/{repo}` | +| `build-results` | Fanout | Distributes `BuildResult` and `QueuedBuildJobs` to all interested consumers | +| `stats` | Fanout | Distributes `EventMessage` metrics payloads | + +### Queues + +| Queue | Type | Exchange Binding | Routing Key | Consumer | +|-------|------|------------------|-------------|----------| +| `build-inputs` | Durable | `github-events` | `issue_comment.*` | `github-comment-filter` | +| `github-events-unknown` | Durable | `github-events` | `unknown.*` | — (no consumer) | +| `mass-rebuild-check-inputs` | Durable | `github-events` | `pull_request.*` (webhook-receiver), `pull_request.nixos/*` (evaluation-filter) | `evaluation-filter` | +| `mass-rebuild-check-jobs` | Durable | Default exchange | `mass-rebuild-check-jobs` (routing key = queue name) | `mass-rebuilder` | +| `build-results` | Durable | `build-results` (fanout) | — | `github-comment-poster` | +| `stats-events` | Durable | `stats` (fanout) | — | `stats` | + +## RabbitMQ Graph + +```mermaid +graph + classDef component fill:#f96 + classDef exc fill:#08f108 + + subgraph Legend + app-example(This is an application):::component + exchange-example{{This is a RabbitMQ exchange}}:::exc + queue-example[/This is a RabbitMQ queue/] + end + + github-webhook-receiver(github-webhook-receiver):::component + github-events{{github-events}}:::exc + build-inputs[/build-inputs/] + github-events-unknown[/github-events-unknown/] + mass-rebuild-check-inputs[/mass-rebuild-check-inputs/] + github-comment-filter(github-comment-filter):::component + evaluation-filter(evaluation-filter):::component + mass-rebuild-check-jobs[/mass-rebuild-check-jobs/] + mass-rebuilder(mass-rebuilder):::component + build-results{{build-results}}:::exc + build-results-queue[/build-results/] + github-comment-poster(github-comment-poster):::component + stats{{stats}}:::exc + stats-events[/stats-events/] + stats-rs(stats):::component + + github-webhook-receiver --> github-events + github-events -->|issue_comment.*| build-inputs + github-events -->|unknown.*| github-events-unknown + github-events -->|pull_request.*| mass-rebuild-check-inputs + build-inputs --> github-comment-filter + github-comment-filter --> mass-rebuild-check-jobs + mass-rebuild-check-inputs --> evaluation-filter + evaluation-filter --> mass-rebuild-check-jobs + mass-rebuild-check-jobs --> mass-rebuilder + build-results --> build-results-queue + build-results-queue --> github-comment-poster + stats --> stats-events + stats-events --> stats-rs +``` + +## Message Types + +| Message | Serialization | Fields | Published To | +|---------|---------------|--------|--------------| +| `BuildJob` | JSON | `repo`, `pr`, `system`, `attrs`, `checkout` | `build-inputs-{system}` queues (via default exchange) | +| `EvaluationJob` | JSON | `repo`, `pr`, `system` | `mass-rebuild-check-jobs` queue | +| `QueuedBuildJobs` | JSON | `repo`, `pr`, `attempt_id`, `total_jobs` | `build-results` exchange | +| `BuildResult` | JSON | `repo`, `pr`, `attempt_id`, `status` (Success/Failure), `system`, `attrs` | `build-results` exchange | +| `EventMessage` | JSON | `event` (tagged enum), `repo`, `pr`, `value` | `stats` exchange | +| Raw webhook JSON | JSON | Raw GitHub webhook payload | `github-events` exchange | diff --git a/Cargo.lock b/Cargo.lock index 3429dcab..2b800c5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -22,11 +22,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "amq-protocol" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95442d3faf08b9bed7491ceda36b0e0268126cb20d44e6e7573cf930d9aa5073" +checksum = "2eab68e8836c5812a01b34c5364d28db50bf686b442e902d9d93e4472318d86e" dependencies = [ "amq-protocol-tcp", "amq-protocol-types", @@ -38,9 +44,9 @@ dependencies = [ [[package]] name = "amq-protocol-tcp" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc754c77e62e58ea7323c3f257a97f747ac57f0ff3f7a91185f484dc1cb25bd2" +checksum = "8689f976dbd9864922f4f53e01ad2b43700ed3eb1bb5667b260522f291434dae" dependencies = [ "amq-protocol-uri", "async-rs", @@ -51,9 +57,9 @@ dependencies = [ [[package]] name = "amq-protocol-types" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8cb8fa210da5ec52799abf7ce45e2275b5a3f82ec4efafcd3e8161b7dbc9a8" +checksum = "27894e9e57d07f701251aeee3d2ab3d7d114bc830bf722d6626027eb55f3b222" dependencies = [ "cookie-factory", "nom 8.0.0", @@ -63,9 +69,9 @@ dependencies = [ [[package]] name = "amq-protocol-uri" -version = "10.2.0" +version = "10.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bc3e8b145641e20b57777a17b3bd7ba6bbdbb2b2ae2f26f6224be35e42c330" +checksum = "64fd5ca63f1b8cba2309aaec8595483c36f3a671225d5ab9fe8265adb9213514" dependencies = [ "amq-protocol-types", "percent-encoding", @@ -81,17 +87,88 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "asn1-rs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -99,7 +176,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror 2.0.18", + "thiserror", "time", ] @@ -151,6 +228,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-executor" version = "1.14.0" @@ -192,14 +281,13 @@ dependencies = [ [[package]] name = "async-rs" -version = "0.8.4" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ebdd144e4199c67b5134fe2280e40d9656071f11df525d3322b43b6534c714" +checksum = "3cd5147201b63ba6883ffabca3a153822f71541748d7108e3e799beaeb283131" dependencies = [ "async-compat", "async-global-executor", "async-trait", - "cfg-if", "futures-core", "futures-io", "hickory-resolver", @@ -232,15 +320,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "zeroize", @@ -248,9 +336,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", @@ -258,6 +346,49 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -268,16 +399,10 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "base64" -version = "0.21.7" +name = "base16ct" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "base64" @@ -297,6 +422,20 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -345,15 +484,51 @@ checksum = "c3adb80ee272c844254166ea32c8ae11c211b3639a293fdde41b1645b6be2c62" [[package]] name = "bumpalo" -version = "3.20.2" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "camino" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" +dependencies = [ + "serde_core", +] + +[[package]] +name = "cargo-platform" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0061da739915fae12ea00e16397555ed4371a6bb285431aab930f61b0aa4ba" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "cargo_metadata" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef987d17b0a113becdd19d3d0022d04d7ef41f9efe4f3fb63ac44ba61df3ade9" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] [[package]] name = "cbc" @@ -366,9 +541,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", "jobserver", @@ -376,12 +551,6 @@ dependencies = [ "shlex", ] -[[package]] -name = "cesu8" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" - [[package]] name = "cfg-if" version = "1.0.4" @@ -412,7 +581,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", + "serde", + "wasm-bindgen", "windows-link", ] @@ -422,10 +594,50 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common 0.1.6", "inout", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmake" version = "0.1.58" @@ -437,9 +649,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "cms" @@ -453,6 +665,23 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -463,6 +692,23 @@ dependencies = [ "memchr", ] +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "compression-core", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -484,6 +730,21 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cookie-factory" version = "0.3.3" @@ -564,11 +825,23 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", "typenum", @@ -576,9 +849,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "hybrid-array", ] @@ -592,11 +865,100 @@ dependencies = [ "cmov", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxx" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" +dependencies = [ + "cc", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" +dependencies = [ + "clap", + "codespan-reporting", + "indexmap", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der" @@ -645,6 +1007,29 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", + "unicode-xid", +] + [[package]] name = "des" version = "0.8.1" @@ -661,27 +1046,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "crypto-common 0.1.7", + "const-oid 0.9.6", + "crypto-common 0.1.6", "subtle", ] [[package]] name = "digest" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", "const-oid 0.10.2", - "crypto-common 0.2.1", + "crypto-common 0.2.2", "ctutils", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", @@ -694,11 +1080,79 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "hkdf", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] [[package]] name = "equivalent" @@ -743,12 +1197,34 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flagset" version = "0.4.7" @@ -763,7 +1239,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ "futures-core", "futures-sink", - "spin 0.9.8", + "spin", ] [[package]] @@ -778,6 +1254,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -787,6 +1269,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fs2" version = "0.4.3" @@ -917,12 +1409,13 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -966,18 +1459,29 @@ dependencies = [ "wasip3", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.0", + "http", "indexmap", "slab", "tokio", @@ -985,20 +1489,158 @@ dependencies = [ "tracing", ] +[[package]] +name = "harmonia-store-aterm" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "bytes", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "memchr", + "serde_json", + "thiserror", +] + +[[package]] +name = "harmonia-store-content-address" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "derive_more", + "harmonia-store-path", + "harmonia-utils-hash", + "serde", + "thiserror", +] + +[[package]] +name = "harmonia-store-derivation" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "bytes", + "data-encoding", + "derive_more", + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", + "serde_json", + "thiserror", + "zerocopy", +] + +[[package]] +name = "harmonia-store-nar-info" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", +] + +[[package]] +name = "harmonia-store-path" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "derive_more", + "harmonia-utils-base-encoding", + "harmonia-utils-hash", + "serde", + "thiserror", + "zerocopy", +] + +[[package]] +name = "harmonia-store-path-info" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "harmonia-store-content-address", + "harmonia-store-path", + "harmonia-utils-hash", + "harmonia-utils-signature", + "serde", +] + +[[package]] +name = "harmonia-utils-base-encoding" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "data-encoding", + "derive_more", + "serde", +] + +[[package]] +name = "harmonia-utils-hash" +version = "0.0.0-alpha.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "blake3", + "data-encoding", + "derive_more", + "harmonia-utils-base-encoding", + "md5", + "serde", + "sha1 0.11.0", + "sha2 0.11.0", + "thiserror", + "tokio", +] + +[[package]] +name = "harmonia-utils-signature" +version = "3.1.0" +source = "git+https://github.com/nix-community/harmonia.git#74fd041be7d57c820087f6b440285dda50a8e7d4" +dependencies = [ + "data-encoding", + "ed25519-dalek", + "getrandom 0.4.2", + "harmonia-utils-base-encoding", + "serde", + "subtle", + "thiserror", + "zeroize", +] + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -1014,9 +1656,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-net" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c61c8db47fae51ba9f8f2a2748bd87542acfbe22f2ec9cf9c8ec72d1ee6e9a6" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" dependencies = [ "async-trait", "cfg-if", @@ -1027,9 +1669,9 @@ dependencies = [ "hickory-proto", "idna", "ipnet", - "jni 0.22.4", + "jni", "rand 0.10.1", - "thiserror 2.0.18", + "thiserror", "tinyvec", "tokio", "tracing", @@ -1038,19 +1680,19 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a916d0494600d99ecb15aadfab677ad97c4de559e8f1af0c129353a733ac1fcc" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" dependencies = [ "data-encoding", "idna", "ipnet", - "jni 0.22.4", + "jni", "once_cell", "prefix-trie", "rand 0.10.1", - "ring 0.17.14", - "thiserror 2.0.18", + "ring", + "thiserror", "tinyvec", "tracing", "url", @@ -1058,9 +1700,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10bd64d950b4d38ca21e25c8ae230712e4955fb8290cfcb29a5e5dc6017e544" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" dependencies = [ "cfg-if", "futures-util", @@ -1068,7 +1710,7 @@ dependencies = [ "hickory-proto", "ipconfig", "ipnet", - "jni 0.22.4", + "jni", "moka", "ndk-context", "once_cell", @@ -1077,11 +1719,20 @@ dependencies = [ "resolv-conf", "smallvec", "system-configuration", - "thiserror 2.0.18", + "thiserror", "tokio", "tracing", ] +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1097,25 +1748,14 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest 0.11.2", -] - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", + "digest 0.11.3", ] [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes", "itoa", @@ -1128,7 +1768,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http", ] [[package]] @@ -1139,7 +1779,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http", "http-body", "pin-project-lite", ] @@ -1157,47 +1797,86 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] -name = "hubcaps" -version = "0.6.2" -source = "git+https://github.com/ofborg/hubcaps.git?rev=0d7466ef941a7a8e160c071e2846e56b90b6ea86#0d7466ef941a7a8e160c071e2846e56b90b6ea86" +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ - "base64 0.22.1", - "data-encoding", + "typenum", +] + +[[package]] +name = "hydra-evaluator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "fs-err", "futures", - "http 1.4.0", - "hyperx", - "jsonwebtoken", - "log", - "mime", - "percent-encoding", - "reqwest", - "serde", - "serde_derive", + "harmonia-store-path", + "hydra-proto", + "hydra-tracing", + "hyper-rustls", + "hyper-util", + "lapin", + "nix-utils", + "ofborg", "serde_json", + "store-transfer", + "tokio", + "tokio-stream", + "toml", + "tonic", + "tower", + "tracing", "url", ] [[package]] -name = "hybrid-array" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +name = "hydra-proto" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "fs-err", + "harmonia-store-content-address", + "harmonia-store-nar-info", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "nix-support", + "prost", + "sha2 0.10.9", + "store-path-utils", + "thiserror", + "tonic", + "tonic-prost", + "tonic-prost-build", +] + +[[package]] +name = "hydra-tracing" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" dependencies = [ - "typenum", + "thiserror", + "tracing", + "tracing-log", + "tracing-subscriber", ] [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "eb92f162bf56536459fc83c79b974bb12837acfed43d6bc370a7916d0ae15ecc" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.4.0", + "http", "http-body", "httparse", "httpdate", @@ -1210,30 +1889,47 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ - "http 1.4.0", + "futures-util", + "http", "hyper", "hyper-util", + "log", "rustls", + "rustls-native-certs", + "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", - "http 1.4.0", + "http", "http-body", "hyper", "ipnet", @@ -1241,24 +1937,12 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", + "tower-layer", "tower-service", "tracing", -] - -[[package]] -name = "hyperx" -version = "1.4.0" -source = "git+https://github.com/chantra/hyperx.git?branch=semver#69f17cf858573db42c2baaf0bfead54521de32f9" -dependencies = [ - "base64 0.13.1", - "bytes", - "http 0.2.12", - "httpdate", - "language-tags", - "mime", - "percent-encoding", - "unicase", + "windows-registry", ] [[package]] @@ -1386,9 +2070,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1401,7 +2085,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1439,13 +2123,18 @@ dependencies = [ ] [[package]] -name = "iri-string" -version = "0.7.12" +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ - "memchr", - "serde", + "either", ] [[package]] @@ -1454,22 +2143,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jni" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" -dependencies = [ - "cesu8", - "cfg-if", - "combine", - "jni-sys 0.3.1", - "log", - "thiserror 1.0.69", - "walkdir", - "windows-sys 0.45.0", -] - [[package]] name = "jni" version = "0.22.4" @@ -1479,10 +2152,10 @@ dependencies = [ "cfg-if", "combine", "jni-macros", - "jni-sys 0.4.1", + "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror", "walkdir", "windows-link", ] @@ -1500,15 +2173,6 @@ dependencies = [ "syn", ] -[[package]] -name = "jni-sys" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" -dependencies = [ - "jni-sys 0.4.1", -] - [[package]] name = "jni-sys" version = "0.4.1" @@ -1540,9 +2204,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11" dependencies = [ "cfg-if", "futures-util", @@ -1552,35 +2216,40 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64 0.21.7", + "base64", + "ed25519-dalek", + "getrandom 0.2.17", + "hmac 0.12.1", + "js-sys", + "p256", + "p384", "pem", - "ring 0.16.20", + "rand 0.8.6", + "rsa", "serde", "serde_json", + "sha2 0.10.9", + "signature", "simple_asn1", + "zeroize", ] -[[package]] -name = "language-tags" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" - [[package]] name = "lapin" -version = "4.5.0" +version = "4.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46ebc501bdffc4d35133680f1ee6d620c131bba216c49704d850f18a6a6495e" +checksum = "0fd20e01fd92597ca352ca7ceed3c589851ebad279dfcada48aa4d24fd3a7caa" dependencies = [ "amq-protocol", "async-rs", "async-trait", "backon", "cfg-if", + "event-listener", "flume", "futures-core", "futures-io", @@ -1592,6 +2261,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "leb128fmt" @@ -1601,9 +2273,24 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] [[package]] name = "linked-hash-map" @@ -1634,9 +2321,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" [[package]] name = "lru-cache" @@ -1662,6 +2349,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md5" version = "0.8.0" @@ -1670,9 +2363,9 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" [[package]] name = "mime" @@ -1688,9 +2381,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -1714,12 +2407,64 @@ dependencies = [ "uuid", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "ndk-context" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" +[[package]] +name = "nix-support" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "fs-err", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-utils-hash", + "regex", + "sha2 0.10.9", + "store-path-utils", + "tokio", + "tracing", +] + +[[package]] +name = "nix-utils" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "bytes", + "cxx", + "cxx-build", + "fs-err", + "futures", + "harmonia-store-aterm", + "harmonia-store-content-address", + "harmonia-store-derivation", + "harmonia-store-path", + "harmonia-store-path-info", + "harmonia-utils-hash", + "harmonia-utils-signature", + "hashbrown 0.16.1", + "pkg-config", + "serde", + "serde_json", + "smallvec", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + [[package]] name = "nom" version = "7.1.3" @@ -1758,11 +2503,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.6", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -1773,6 +2534,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1780,34 +2552,76 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", +] + +[[package]] +name = "octocrab" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2ad8abffe4e2b05f9cdc7e061de63d305a6dca0af81ca1064a7d98e0b78267" +dependencies = [ + "arc-swap", + "async-trait", + "base64", + "bytes", + "cargo_metadata", + "cfg-if", + "chrono", + "futures", + "futures-util", + "getrandom 0.2.17", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-timeout", + "hyper-util", + "jsonwebtoken", + "once_cell", + "percent-encoding", + "pin-project", + "secrecy", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "snafu", + "tokio", + "tower", + "tower-http", + "tracing", + "url", + "web-time", ] [[package]] name = "ofborg" version = "0.1.9" dependencies = [ + "anyhow", "async-trait", "brace-expand", + "bytes", "chrono", - "either", "fs2", "futures", - "futures-util", "hex", "hmac 0.13.0", - "http 1.4.0", + "http", + "http-body", "http-body-util", - "hubcaps", "hyper", "hyper-util", + "jsonwebtoken", "lapin", "lru-cache", "md5", - "mime", "nom 8.0.0", + "octocrab", "parking_lot", "regex", - "rustls-pki-types", "serde", "serde_json", "sha2 0.11.0", @@ -1819,12 +2633,30 @@ dependencies = [ "uuid", ] +[[package]] +name = "ofborg-send-event" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "hex", + "hmac 0.13.0", + "octocrab", + "ofborg", + "reqwest", + "serde_json", + "sha2 0.11.0", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "ofborg-simple-build" version = "0.1.0" dependencies = [ - "log", "ofborg", + "tracing", ] [[package]] @@ -1846,6 +2678,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -1858,21 +2696,45 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffb9bf5222606eb712d3bb30e01bc9420545b00859970897e70c682353a034f2" dependencies = [ - "base64 0.22.1", - "cbc", - "cms", - "der", - "des", - "hex", - "hmac 0.12.1", - "pkcs12", - "pkcs5", - "rand 0.10.1", - "rc2", - "sha1", + "base64", + "cbc", + "cms", + "der", + "des", + "hex", + "hmac 0.12.1", + "pkcs12", + "pkcs5", + "rand 0.10.1", + "rc2", + "sha1 0.10.6", + "sha2 0.10.9", + "thiserror", + "x509-parser", +] + +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2 0.10.9", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", "sha2 0.10.9", - "thiserror 2.0.18", - "x509-parser", ] [[package]] @@ -1916,11 +2778,12 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.1" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.13.1", + "base64", + "serde_core", ] [[package]] @@ -1938,6 +2801,37 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1955,6 +2849,17 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + [[package]] name = "pkcs12" version = "0.1.0" @@ -1985,6 +2890,22 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -2017,9 +2938,9 @@ dependencies = [ [[package]] name = "prefix-trie" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23370be78b7e5bcbb0cab4a02047eb040279a693c78daad04c2c5f1c24a83503" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" dependencies = [ "either", "ipnet", @@ -2036,6 +2957,15 @@ dependencies = [ "syn", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -2045,6 +2975,79 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quinn" version = "0.11.9" @@ -2059,7 +3062,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror", "tokio", "tracing", "web-time", @@ -2076,12 +3079,12 @@ dependencies = [ "getrandom 0.3.4", "lru-slab", "rand 0.9.4", - "ring 0.17.14", + "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror", "tinyvec", "tracing", "web-time", @@ -2098,7 +3101,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -2122,13 +3125,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2143,6 +3157,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2153,6 +3177,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -2217,14 +3250,16 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64 0.22.1", + "base64", "bytes", + "encoding_rs", "futures-core", - "http 1.4.0", + "h2", + "http", "http-body", "http-body-util", "hyper", @@ -2232,12 +3267,15 @@ dependencies = [ "hyper-util", "js-sys", "log", + "mime", "percent-encoding", "pin-project-lite", "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier 0.6.2", + "rustls-platform-verifier", + "serde", + "serde_json", "sync_wrapper", "tokio", "tokio-rustls", @@ -2257,18 +3295,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] -name = "ring" -version = "0.16.20" +name = "rfc6979" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "hmac 0.12.1", + "subtle", ] [[package]] @@ -2281,10 +3314,30 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-hash" version = "2.1.2" @@ -2324,11 +3377,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", + "log", "once_cell", "rustls-pki-types", "rustls-webpki", @@ -2338,16 +3392,16 @@ dependencies = [ [[package]] name = "rustls-connector" -version = "0.23.0" +version = "0.23.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a288bf4b9d06a7c33e54e6879e2142fa45fc936017c3e1319147889daedf14d4" +checksum = "b7664e32b0ffbec386bdf1d7cbca51a89551a90d3278f135186cd6cb3cfa889c" dependencies = [ "futures-io", "futures-rustls", "log", "rustls", "rustls-pki-types", - "rustls-platform-verifier 0.7.0", + "rustls-platform-verifier", "rustls-webpki", ] @@ -2365,35 +3419,14 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" -dependencies = [ - "core-foundation 0.10.1", - "core-foundation-sys", - "jni 0.21.1", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - [[package]] name = "rustls-platform-verifier" version = "0.7.0" @@ -2402,7 +3435,7 @@ checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni 0.22.4", + "jni", "log", "once_cell", "rustls", @@ -2423,14 +3456,14 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -2439,6 +3472,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "salsa20" version = "0.10.2" @@ -2472,6 +3511,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + [[package]] name = "scrypt" version = "0.11.0" @@ -2483,6 +3528,29 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2511,6 +3579,10 @@ name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "serde" @@ -2544,9 +3616,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -2555,6 +3627,38 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2566,6 +3670,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2585,7 +3700,7 @@ checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "digest 0.11.2", + "digest 0.11.3", ] [[package]] @@ -2599,9 +3714,29 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] [[package]] name = "simd_cesu8" @@ -2627,7 +3762,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.18", + "thiserror", "time", ] @@ -2641,24 +3776,39 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -2684,6 +3834,42 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "store-path-utils" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "harmonia-store-path", +] + +[[package]] +name = "store-transfer" +version = "0.1.0" +source = "git+https://github.com/helsinki-systems/hydra?rev=55a9f8bfd47380ecbd30ee6649175dab64bf6df7#55a9f8bfd47380ecbd30ee6649175dab64bf6df7" +dependencies = [ + "async-compression", + "bytes", + "futures", + "harmonia-store-path", + "harmonia-store-path-info", + "hashbrown 0.16.1", + "hydra-proto", + "nix-utils", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tracing", + "zstd", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2750,9 +3936,9 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tcp-stream" -version = "0.34.5" +version = "0.34.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6638f031787a4854f0ecc669514404d201a3f4eab1849e4151f01a28324972de" +checksum = "300d0735de48a565461c2ea14cc75b80ee5b6be3b4f5aeabe3553e4c2df8d23d" dependencies = [ "async-rs", "cfg-if", @@ -2775,12 +3961,12 @@ dependencies = [ ] [[package]] -name = "thiserror" -version = "1.0.69" +name = "termcolor" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ - "thiserror-impl 1.0.69", + "winapi-util", ] [[package]] @@ -2789,18 +3975,7 @@ version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.18", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "thiserror-impl", ] [[package]] @@ -2881,14 +4056,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", "mio", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -2939,6 +4116,116 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots", + "zstd", +] + +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", + "tempfile", + "tonic-build", +] + [[package]] name = "tower" version = "0.5.3" @@ -2947,29 +4234,34 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.0", + "http", "http-body", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "tracing", + "url", ] [[package]] @@ -2990,6 +4282,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3083,16 +4376,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "unicode-xid" -version = "0.2.6" +name = "unicode-segmentation" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] -name = "untrusted" -version = "0.7.1" +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "untrusted" @@ -3110,6 +4409,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -3118,6 +4418,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -3186,9 +4492,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409" dependencies = [ "cfg-if", "once_cell", @@ -3199,9 +4505,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3209,9 +4515,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3219,9 +4525,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e" dependencies = [ "bumpalo", "proc-macro2", @@ -3232,9 +4538,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437" dependencies = [ "unicode-ident", ] @@ -3275,9 +4581,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" dependencies = [ "js-sys", "wasm-bindgen", @@ -3290,6 +4596,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", + "serde", "wasm-bindgen", ] @@ -3302,6 +4609,15 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "widestring" version = "1.2.1" @@ -3409,31 +4725,13 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -3445,180 +4743,64 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -3626,10 +4808,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" +name = "winnow" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" [[package]] name = "wit-bindgen" @@ -3755,7 +4937,7 @@ dependencies = [ "nom 7.1.3", "oid-registry", "rusticata-macros", - "thiserror 2.0.18", + "thiserror", "time", ] @@ -3784,18 +4966,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "bce33a6288fa3f072a8c2c7d0f2fdbb90e28298f0135c1f99b96c3db2efcc60b" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "8fd425244944f4ab65ccff928e7323354c5a018c75838362fdce749dfad2ee1e" dependencies = [ "proc-macro2", "quote", @@ -3804,9 +4986,9 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] @@ -3828,6 +5010,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "zerotrie" @@ -3867,3 +5063,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index a1688cec..218e585b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,9 @@ [workspace] members = [ - "ofborg", - "ofborg-simple-build" + "ofborg", + "ofborg-send-event", + "ofborg-simple-build", + "hydra-evaluator", ] resolver = "2" @@ -9,5 +11,4 @@ resolver = "2" debug = true [patch.crates-io] -#hubcaps = { path = "../hubcaps" } #amq-proto = { path = "rust-amq-proto" } diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..3f217db5 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,116 @@ +# Local Development + +## Prerequisites + +- [Nix](https://nixos.org/download.html) (with flakes enabled) +- [Docker](https://docs.docker.com/engine/install/) +- [mprocs](https://github.com/pvolok/mprocs) — available in the dev shell + +## Quick start + +```shell +# Enter the development environment +nix-shell + +# Start all services +mprocs +``` + +This brings up: + +| Process | Autostart | Description | +|---------------------------|-----------|-----------------------------------------------------------------| +| `rabbitmq` | yes | Docker container (`rabbitmq:4-management`) on ports 5672, 15672 | +| `rabbitmq-init` | yes | Waits for RabbitMQ, then creates the `ofborg` vhost/user | +| `evaluation-filter` | yes | Processes evaluation events from the queue | +| `github-webhook-receiver` | yes | HTTP server on `[::1]:9899` — receives GitHub webhooks | +| `mass-rebuilder` | yes | Orchestrates mass rebuild jobs | +| `stats` | yes | Stats server on `[::1]:9898` | +| `github-comment-filter` | yes | Disabled by default; start manually in mprocs | +| `hydra-evaluator` | yes | Consumes `HydraEvalJob` messages, imports derivations into queue-runner via gRPC | + +## What happens under the hood + +1. **`mprocs/start-rabbitmq.sh`** — Starts RabbitMQ via Docker (`rabbitmq:4-management`). Data is persisted in `.ofborg-data/rabbitmq/`. The admin user defaults to `admin` / `admin`. + +2. **`mprocs/wait-for-rabbitmq.sh`** — Polls the RabbitMQ management API healthcheck until the server is ready. + +3. **`mprocs/bootstrap-rabbitmq.sh`** — Creates the `ofborg` vhost, a restricted `ofborg` user with a random password, and writes the password to `.ofborg-data/.amqp-password`. + +4. **`mprocs/bootstrap-ofborg.sh`** — Generates a random webhook secret (`.ofborg-data/.webhook-secret`) and writes a complete config at `.ofborg-data/local.json`. The config wires all services to the local RabbitMQ instance. + +All services then start via `cargo r --bin .ofborg-data/local.json`, so changes to source code are reflected immediately (cargo recompiles on restart). The `hydra-evaluator` crate runs via `cargo r -p hydra-evaluator -- .ofborg-data/local.json`. + +## Hydra evaluator flow + +When the `hydra_evaluator` config section is present, the mass-rebuilder publishes `HydraEvalJob` messages to the `hydra-eval-jobs` queue after a successful evaluation. The `hydra-evaluator` binary then: + +1. Consumes `HydraEvalJob` messages from `hydra-eval-jobs`. +2. Resolves each drv path against the local Nix store. +3. Streams the corresponding NARs (zstd-compressed) to the queue-runner via the `BuildResult` gRPC stream. +4. Calls `CreateBuild` on the queue-runner to register the builds under the configured `jobset_id`. + +If the `hydra_evaluator` config is absent, the mass-rebuilder skips the hydra integration step entirely. + +## Sending test webhook events + +Use the `ofborg-send-event` crate to simulate GitHub pull_request webhooks: + +```shell +# Send PR #123456 from NixOS/nixpkgs to localhost:9899 +cargo run -p ofborg-send-event -- 123456 + +# Use a different repo +cargo run -p ofborg-send-event -- --full-repo-name "ofborg/testpkgs" 42 + +# Custom event type +cargo run -p ofborg-send-event -- --event push 123456 + +# Point at a different webhook receiver +cargo run -p ofborg-send-event -- --webhook-receiver-url http://localhost:9999 123456 + +# Supply the webhook secret (required if the receiver validates signatures) +cargo run -p ofborg-send-event -- --secret-path .ofborg-data/.webhook-secret 123456 + +# Custom delivery ID and extra headers +cargo run -p ofborg-send-event -- --delivery-id my-id --header X-Custom=val 123456 + +# Verbose mode (shows response headers) +cargo run -p ofborg-send-event -- --verbose 123456 +``` + +The tool fetches the real PR data from GitHub, constructs a `PullRequestEvent` payload, signs it with HMAC-SHA256 (if `--secret-path` is given), and POSTs it to the receiver. + +### CLI reference + +``` +Usage: ofborg-send-event [OPTIONS] + +Arguments: + PR that should be fetched + +Options: + --webhook-receiver-url [default: http://localhost:9899] + --event [default: pull_request] + --secret-path Shared secret for X-Hub-Signature-256 + --delivery-id X-GitHub-Delivery header (default: random UUID) + --header Add arbitrary header (repeatable) + --verbose Print response headers + --timeout-secs [default: 30] + --full-repo-name [default: NixOS/nixpkgs] +``` + +## Building and testing + +```shell +# All in the dev shell: + +cargo build # build all binaries +cargo test # run tests +cargo clippy # lint +cargo fmt # format code +cargo r --bin # run a specific binary + +# Full CI check (from repo root): +nix-shell --pure --run checkPhase +``` diff --git a/README.md b/README.md index 9e4dcc37..35b0036f 100644 --- a/README.md +++ b/README.md @@ -6,30 +6,13 @@ 2. Be gentle; try not to run mass rebuilds or massive builds (like Chromium) on it. -## Automatic Building +## Automatic Evaluation -All users will have their PRs automatically trigger builds if their commits +All users will have their PRs automatically evaluated if their commits follow the well-defined format of Nixpkgs. Specifically: prefixing the commit title with the package attribute. This includes package bumps as well as other changes. -Example commit titles and the builds they will start: - -| Message | Automatic Build | -|-----------------------------------------------------------------------|----------------------------------------------------------| -| `vim: 1.0.0 -> 2.0.0` | `vim` | -| `vagrant: Fix dependencies for version 2.0.2 ` | `vagrant` | -| `python36Packages.requests,python27Packages.requests: 1.0.0 -> 2.0.0` | `python36Packages.requests`, `python27Packages.requests` | -| `python{27,310}Packages.requests: 1.0.0 -> 2.0.0` | `python27Packages.requests`, `python310Packages.requests` | - -When opening a PR with multiple commits, ofborg creates a single build job for -all detected packages. If multiple commits get pushed to a PR one-by-one, each -detected package will get a separate build job. - -If the title of a PR begins with `WIP:` or contains `[WIP]` anywhere, its -packages are not built automatically. -**Note**: Marking a PR as a draft does not prevent automatic builds. - ## Commands The comment parser is line-based, so commentary can be interwoven with @@ -230,7 +213,4 @@ This will override the default of `-D warnings` set in [`shell.nix`](./shell.nix), which tells Rust to error if it detects any warnings. -# Running a builder -If you want to run a builder of your own, check out the [wiki page on operating -a builder](https://github.com/NixOS/ofborg/wiki/Operating-a-Builder/). diff --git a/flake.lock b/flake.lock index 5d203907..5e773697 100644 --- a/flake.lock +++ b/flake.lock @@ -16,9 +16,26 @@ "type": "github" } }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1778729098, + "narHash": "sha256-17SbusskVZng4nwevRqsWNJf27nMG7UczvtgWTUJttg=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "39ea44cddd5060b8cd413ed5e13c6af61f302283", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable-small", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" } } }, diff --git a/flake.nix b/flake.nix index 1a9f2b26..ab569579 100644 --- a/flake.nix +++ b/flake.nix @@ -1,12 +1,14 @@ { inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-25.11"; + nixpkgs-unstable.url = "github:nixos/nixpkgs/nixos-unstable-small"; }; outputs = { self, nixpkgs, + nixpkgs-unstable, ... }@inputs: let @@ -26,6 +28,10 @@ pkgs = import nixpkgs { inherit system; }; + unstable = import nixpkgs-unstable { + inherit system; + }; + nix = unstable.nixVersions.nix_2_34; in { default = pkgs.mkShell { @@ -40,6 +46,14 @@ rustfmt pkg-config git + mprocs + + zlib + protobuf + + nlohmann_json + libsodium + boost ]; buildInputs = with pkgs; @@ -69,8 +83,9 @@ RUSTFLAGS = "-D warnings"; RUST_BACKTRACE = "1"; - RUST_LOG = "ofborg=debug"; + RUST_LOG = "ofborg=debug,info"; NIX_PATH = "nixpkgs=${pkgs.path}"; + NIX_CFLAGS_COMPILE = "-Wno-error"; }; } ); @@ -108,7 +123,6 @@ cargoLock = { lockFile = ./Cargo.lock; outputHashes = { - "hubcaps-0.6.2" = "sha256-Vl4wQIKQVRxkpQxL8fL9rndAN3TKLV4OjgnZOpT6HRo="; "hyperx-1.4.0" = "sha256-MW/KxxMYvj/DYVKrYa7rDKwrH6s8uQOCA0dR2W7GBeg="; }; }; diff --git a/hydra-evaluator/Cargo.toml b/hydra-evaluator/Cargo.toml new file mode 100644 index 00000000..19e2dcbd --- /dev/null +++ b/hydra-evaluator/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "hydra-evaluator" +version = "0.1.0" +edition = "2024" +license = "GPL-3.0" + +[dependencies] +tracing = "0.1" + +anyhow = "1.0.98" +clap = { version = "4", features = ["derive"] } +fs-err = { version = "3.0", features = ["tokio"] } + +tokio = { version = "1.50", features = ["full"] } +tokio-stream = "0.1" +futures = "0.3" +tonic = { version = "0.14", features = ["zstd", "tls-webpki-roots"] } +tower = "0.5" +hyper-util = { version = "0.1.10", features = ["full"] } + +lapin = "4.3.0" +serde_json = "1.0" + +url = "2.5.4" +toml = "1.0.0" + +harmonia-store-path = { git = "https://github.com/nix-community/harmonia.git" } +nix-utils = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7" } +hydra-tracing = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7" } +hydra-proto = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7", features = [ "client" ] } +store-transfer = { git = "https://github.com/helsinki-systems/hydra", rev = "55a9f8bfd47380ecbd30ee6649175dab64bf6df7" } +ofborg = { path = "../ofborg" } +hyper-rustls = "=0.27.3" # required for ofborg::octocrab diff --git a/hydra-evaluator/src/config.rs b/hydra-evaluator/src/config.rs new file mode 100644 index 00000000..2d88e88a --- /dev/null +++ b/hydra-evaluator/src/config.rs @@ -0,0 +1,128 @@ +use clap::Parser; + +#[derive(Parser, Debug)] +#[clap( + author, + version, + about, + long_about = "ofborg-evaluator: injects derivations into a queue-runner jobset" +)] +pub struct Cli { + /// Queue-runner gRPC endpoint + #[clap(short, long, default_value = "http://[::1]:50051")] + pub gateway_endpoint: String, + + /// File containing the bearer token for authentication + #[clap(long)] + pub authorization_file: Option, + + /// Whether to use mTLS + #[clap(long)] + pub mtls: bool, + + /// Path to Server root CA cert + #[clap(long)] + pub server_root_ca_cert_path: Option, + + /// Path to Client cert + #[clap(long)] + pub client_cert_path: Option, + + /// Path to Client key + #[clap(long)] + pub client_key_path: Option, + + /// Domain name for mTLS + #[clap(long)] + pub domain_name: Option, + + /// Config file path + #[clap()] + pub config_path: std::path::PathBuf, +} + +impl Cli { + #[must_use] + pub fn new() -> Self { + Self::parse() + } + + pub async fn get_authorization_token(&self) -> anyhow::Result> { + let Some(path) = &self.authorization_file else { + return Ok(None); + }; + + let content = fs_err::tokio::read_to_string(path).await?; + + // Try parsing as TOML and extracting the "token" field. + // This allows reusing the same token file format as the queue-runner. + if let Ok(value) = content.parse::() + && let Some(token) = value.get("token").and_then(toml::Value::as_str) + { + return Ok(Some(token.to_string())); + } + + // Fall back: treat entire file content as a plain-text token + Ok(Some(content.trim().to_string())) + } + + #[must_use] + pub const fn mtls_enabled(&self) -> bool { + self.mtls + } + + #[must_use] + pub fn mtls_configured_correctly(&self) -> bool { + if self.mtls { + self.server_root_ca_cert_path.is_some() + && self.client_cert_path.is_some() + && self.client_key_path.is_some() + && self.domain_name.is_some() + } else { + self.server_root_ca_cert_path.is_none() + && self.client_cert_path.is_none() + && self.client_key_path.is_none() + && self.domain_name.is_none() + } + } + + pub async fn get_mtls( + &self, + ) -> anyhow::Result<( + tonic::transport::Certificate, + tonic::transport::Identity, + String, + )> { + let server_root_ca_cert_path = self + .server_root_ca_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("server_root_ca_cert_path not provided"))?; + let client_cert_path = self + .client_cert_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_cert_path not provided"))?; + let client_key_path = self + .client_key_path + .as_deref() + .ok_or_else(|| anyhow::anyhow!("client_key_path not provided"))?; + let domain_name = self + .domain_name + .as_deref() + .ok_or_else(|| anyhow::anyhow!("domain_name not provided"))?; + + let server_root_ca_cert = fs_err::tokio::read_to_string(server_root_ca_cert_path).await?; + let server_root_ca_cert = tonic::transport::Certificate::from_pem(server_root_ca_cert); + + let client_cert = fs_err::tokio::read_to_string(client_cert_path).await?; + let client_key = fs_err::tokio::read_to_string(client_key_path).await?; + let client_identity = tonic::transport::Identity::from_pem(client_cert, client_key); + + Ok((server_root_ca_cert, client_identity, domain_name.to_owned())) + } +} + +impl Default for Cli { + fn default() -> Self { + Self::new() + } +} diff --git a/hydra-evaluator/src/grpc.rs b/hydra-evaluator/src/grpc.rs new file mode 100644 index 00000000..ed0c152d --- /dev/null +++ b/hydra-evaluator/src/grpc.rs @@ -0,0 +1,107 @@ +use anyhow::Context as _; +use tonic::Request; +use tonic::transport::Channel; + +use hydra_proto::runner_service_client::RunnerServiceClient; + +#[derive(Debug, Clone)] +pub enum AuthInterceptor { + Token { + token: tonic::metadata::MetadataValue, + }, + Noop, +} + +impl tonic::service::Interceptor for AuthInterceptor { + fn call(&mut self, mut request: Request<()>) -> Result, tonic::Status> { + if let Self::Token { token } = self { + request + .metadata_mut() + .insert("authorization", token.clone()); + } + + Ok(request) + } +} + +pub type OfborgClient = + RunnerServiceClient>; + +#[tracing::instrument(err)] +pub async fn init_client(cli: &crate::config::Cli) -> anyhow::Result { + if !cli.mtls_configured_correctly() { + tracing::error!( + "mtls configured improperly, please pass all options: \ + server_root_ca_cert_path, client_cert_path, client_key_path and domain_name!" + ); + return Err(anyhow::anyhow!("Configuration issue")); + } + + tracing::info!("connecting to {}", cli.gateway_endpoint); + let channel = if cli.mtls_enabled() { + tracing::info!("mtls is enabled"); + let (server_root_ca_cert, client_identity, domain_name) = cli + .get_mtls() + .await + .context("Failed to get_mtls Certificate and Identity")?; + let tls = tonic::transport::ClientTlsConfig::new() + .domain_name(domain_name) + .ca_certificate(server_root_ca_cert) + .identity(client_identity); + + Channel::builder(cli.gateway_endpoint.parse()?) + .tls_config(tls) + .context("Failed to attach tls config")? + .connect() + .await + .context("Failed to establish connection with Channel")? + } else if let Some(path) = cli.gateway_endpoint.strip_prefix("unix://") { + let path = path.to_owned(); + tonic::transport::Endpoint::try_from("http://[::]:50051")? + .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { + let path = path.clone(); + async move { + Ok::<_, std::io::Error>(hyper_util::rt::TokioIo::new( + tokio::net::UnixStream::connect(&path).await?, + )) + } + })) + .await + .context("Failed to establish unix socket connection with Channel")? + } else if cli.gateway_endpoint.starts_with("https://") { + let uri: url::Url = cli + .gateway_endpoint + .parse() + .context("Failed to parse gateway_endpoint")?; + + let tls = tonic::transport::ClientTlsConfig::new() + .domain_name( + uri.domain() + .ok_or_else(|| anyhow::anyhow!("No domain_name found for gateway_endpoint"))?, + ) + .with_enabled_roots(); + Channel::builder(cli.gateway_endpoint.parse()?) + .tls_config(tls) + .context("Failed to attach tls config")? + .connect() + .await + .context("Failed to establish connection with Channel")? + } else { + Channel::builder(cli.gateway_endpoint.parse()?) + .connect() + .await + .context("Failed to establish connection with Channel")? + }; + + let interceptor = if let Some(t) = cli.get_authorization_token().await? { + AuthInterceptor::Token { + token: format!("Bearer {t}").parse()?, + } + } else { + AuthInterceptor::Noop + }; + + Ok(RunnerServiceClient::with_interceptor(channel, interceptor) + .max_decoding_message_size(50 * 1024 * 1024) + .max_encoding_message_size(50 * 1024 * 1024)) +} diff --git a/hydra-evaluator/src/main.rs b/hydra-evaluator/src/main.rs new file mode 100644 index 00000000..10b4f6b4 --- /dev/null +++ b/hydra-evaluator/src/main.rs @@ -0,0 +1,221 @@ +#![forbid(unsafe_code)] +#![deny( + clippy::all, + clippy::pedantic, + clippy::expect_used, + clippy::unwrap_used, + future_incompatible, + missing_debug_implementations, + nonstandard_style, + missing_copy_implementations, + unused_qualifications +)] +#![allow(clippy::missing_errors_doc)] + +mod config; +mod grpc; + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::Context as _; +use futures::TryFutureExt as _; +use harmonia_store_path::{FromStoreDirStr, StorePath}; +use hydra_proto::{CreateBuildRequest, ProtoStorePath}; +use lapin::options::{BasicAckOptions, BasicConsumeOptions, QueueDeclareOptions}; +use lapin::types::FieldTable; +use nix_utils::BaseStore as _; +use tokio::sync::mpsc; +use tokio_stream::StreamExt as _; + +use crate::grpc::OfborgClient; + +#[tracing::instrument(skip(client, drv_paths), err)] +async fn import_drvs(client: &mut OfborgClient, drv_paths: &[StorePath]) -> anyhow::Result<()> { + let (tx, rx) = + mpsc::unbounded_channel::>(); + + let store = nix_utils::LocalStore::init(); + let drv_paths = drv_paths.to_vec(); + let sender = tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Runtime::new()?; + let infos = rt.block_on(store.query_path_infos(&drv_paths.iter().collect::>()))?; + store_transfer::export::export(&store, &drv_paths, &infos, &tx); + Ok::<(), anyhow::Error>(()) + }); + + let upload = client + .build_result(tokio_stream::StreamExt::filter_map( + tokio_stream::wrappers::UnboundedReceiverStream::new(rx), + Result::ok, + )) + .map_err(Into::::into); + + let (upload_result, sender_result) = futures::future::join(upload, sender).await; + upload_result?; + sender_result??; + + Ok(()) +} + +#[tracing::instrument(skip(client), err)] +async fn create_builds( + client: &mut OfborgClient, + jobset_id: i32, + drv_paths: &[StorePath], +) -> anyhow::Result> { + let response = client + .create_build(CreateBuildRequest { + jobset_id, + drv_paths: drv_paths.iter().map(ProtoStorePath::from).collect(), + }) + .await + .context("Failed to call CreateBuild")?; + + Ok(response.into_inner().build_ids) +} + +#[tokio::main] +#[allow(clippy::too_many_lines)] +async fn main() -> anyhow::Result<()> { + hydra_tracing::init()?; + nix_utils::init_nix(); + + let cli = Arc::new(config::Cli::new()); + let Some(cfg) = ofborg::config::load(&cli.config_path).hydra_evaluator else { + tracing::error!("No ofborg/hydra evaluator configuration found!"); + panic!(); + }; + + tracing::info!( + "ofborg-evaluator starting endpoint={}", + cli.gateway_endpoint + ); + + tracing::info!("running in AMQP consumer mode"); + let conn = ofborg::easylapin::from_config(&cfg.rabbitmq).await?; + let chan = conn.create_channel().await?; + + // Declare the hydra-eval-jobs queue (must match what mass-rebuilder publishes to) + chan.queue_declare( + "hydra-eval-jobs".into(), + QueueDeclareOptions { + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + nowait: false, + }, + FieldTable::default(), + ) + .await?; + + tracing::info!("connecting to queue-runner gRPC"); + let mut client = grpc::init_client(&cli).await?; + + tracing::info!("consuming from hydra-eval-jobs"); + let mut consumer = chan + .basic_consume( + "hydra-eval-jobs".into(), + "ofborg-hydra-evaluator".into(), + BasicConsumeOptions::default(), + FieldTable::default(), + ) + .await?; + + while let Some(Ok(delivery)) = consumer.next().await { + let body = &delivery.data; + let job: ofborg::message::hydra_eval_job::HydraEvalJob = match serde_json::from_slice(body) + { + Ok(job) => job, + Err(e) => { + tracing::error!( + "Failed to deserialize HydraEvalJob: {e}, body: {:?}", + std::str::from_utf8(body) + ); + let _ = chan + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await; + continue; + } + }; + + tracing::info!( + "Processing HydraEvalJob for {}/{} PR #{} ({} drv paths, jobset_id={})", + job.repo.owner, + job.repo.name, + job.pr.number, + job.drv_paths.len(), + job.jobset_id, + ); + + if job.drv_paths.is_empty() { + tracing::warn!("Received HydraEvalJob with no drv paths, acking"); + let _ = chan + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await; + continue; + } + + let store_dir = nix_utils::LocalStore::init().store_dir().clone(); + let drv_paths: Vec = job + .drv_paths + .iter() + .map(|s| { + StorePath::from_store_dir_str(&store_dir, s) + .unwrap_or_else(|e| panic!("Invalid store path '{s}': {e}")) + }) + .collect(); + + match import_drvs(&mut client, &drv_paths).await { + Ok(()) => { + tracing::info!("Successfully imported {} drv(s)", drv_paths.len()); + } + Err(e) => { + tracing::error!("Failed to import drvs: {e:?}"); + // Nack and requeue so another consumer can retry + let _ = chan + .basic_nack( + delivery.delivery_tag, + lapin::options::BasicNackOptions { + requeue: true, + ..Default::default() + }, + ) + .await; + continue; + } + } + + match create_builds(&mut client, job.jobset_id, &drv_paths).await { + Ok(build_ids) => { + tracing::info!("Created {} build(s)", build_ids.len()); + for (drv_path, build_id) in &build_ids { + tracing::info!(" {build_id} <- {drv_path}"); + } + } + Err(e) => { + tracing::error!("Failed to create builds: {e:?}"); + let _ = chan + .basic_nack( + delivery.delivery_tag, + lapin::options::BasicNackOptions { + requeue: true, + ..Default::default() + }, + ) + .await; + continue; + } + } + + let _ = chan + .basic_ack(delivery.delivery_tag, BasicAckOptions::default()) + .await; + tracing::info!("Finished processing job for PR #{}", job.pr.number); + } + + drop(conn); // Close connection. + tracing::info!("Closed the session... EOF"); + Ok(()) +} diff --git a/mprocs.yaml b/mprocs.yaml new file mode 100644 index 00000000..718bae25 --- /dev/null +++ b/mprocs.yaml @@ -0,0 +1,35 @@ +procs: + rabbitmq: + shell: mprocs/start-rabbitmq.sh + env: + RUST_LOG: info + stop: + send-keys: [""] + rabbitmq-init: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-rabbitmq.sh + env: + RUST_LOG: info + evaluation-filter: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin evaluation-filter .ofborg-data/local.json + env: + RUST_LOG: info + github-comment-filter: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin github-comment-filter .ofborg-data/local.json + env: + RUST_LOG: info + github-webhook-receiver: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin github-webhook-receiver .ofborg-data/local.json + env: + RUST_LOG: info + mass-rebuilder: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin mass-rebuilder .ofborg-data/local.json + env: + RUST_LOG: info + stats: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r --bin stats .ofborg-data/local.json + env: + RUST_LOG: info + hydra-evaluator: + shell: mprocs/wait-for-rabbitmq.sh && mprocs/bootstrap-ofborg.sh && cargo r -p hydra-evaluator -- .ofborg-data/local.json + env: + RUST_LOG: info diff --git a/mprocs/bootstrap-ofborg.sh b/mprocs/bootstrap-ofborg.sh new file mode 100755 index 00000000..431929f8 --- /dev/null +++ b/mprocs/bootstrap-ofborg.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DATA_DIR="${DATA_DIR:-$(pwd)/.ofborg-data}" +mkdir -pv "${DATA_DIR}" + +gen_secret() { + openssl rand -base64 18 | tr -d '/+=' | head -c 24 +} +if [[ ! -f "${DATA_DIR}/.webhook-secret" ]]; then + gen_secret > "${DATA_DIR}/.webhook-secret" +fi + +cat < .ofborg-data/local.json +{ + "github_webhook_receiver": { + "listen": "[::1]:9899", + "webhook_secret_file": "${DATA_DIR}/.webhook-secret", + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "mass_rebuilder": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "evaluation_filter": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "github_comment_filter": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "hydra_evaluator": { + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + }, + "gateway_endpoint": "[::1]:50051", + "jobset_id": 7 + }, + "stats": { + "listen": "[::1]:9898", + "rabbitmq": { + "host": "localhost:5672", + "ssl": false, + "username": "ofborg", + "password_file": "${DATA_DIR}/.amqp-password", + "virtualhost": "ofborg" + } + }, + "runner": { + "identity": "...", + "repos": [ + "ofborg/testpkgs" + ], + "disable_trusted_users": true + }, + "checkout": { + "root": "${DATA_DIR}/checkout" + }, + "nix": { + "system": "x86_64-linux", + "remote": "daemon", + "build_timeout_seconds": 3600, + "initial_heap_size": "4g" + } +} +EOF diff --git a/mprocs/bootstrap-rabbitmq.sh b/mprocs/bootstrap-rabbitmq.sh new file mode 100755 index 00000000..63442d2e --- /dev/null +++ b/mprocs/bootstrap-rabbitmq.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -euo pipefail + +RABBITMQ_HOST="${RABBITMQ_HOST:-localhost}" +RABBITMQ_MGMT_PORT="${RABBITMQ_MGMT_PORT:-15672}" +ADMIN_USER="${RABBITMQ_ADMIN_USER:-admin}" +ADMIN_PASS="${RABBITMQ_ADMIN_PASS:-admin}" +CONFIG_OUT="${CONFIG_OUT:-$(pwd)/.ofborg-data/rabbitmq-config.json}" + +BASE_URL="http://${RABBITMQ_HOST}:${RABBITMQ_MGMT_PORT}/api" + +rmq() { + set -x + local method="$1" path="$2"; shift 2 + curl -sf -u "${ADMIN_USER}:${ADMIN_PASS}" \ + -X "${method}" \ + -H "Content-Type: application/json" \ + "${BASE_URL}${path}" "$@" +} + +urlencode() { + python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1],safe=''))" "$1" +} + +gen_secret() { + openssl rand -base64 18 | tr -d '/+=' | head -c 24 +} + +mkdir -p "$(dirname "${CONFIG_OUT}")" "$(pwd)/.ofborg-data/rabbitmq" + +rmq PUT "/vhosts/$(urlencode "ofborg")" -d '{}' >/dev/null + +pass="$(gen_secret)" +rmq PUT "/users/ofborg" -d "{\"password\":\"${pass}\", \"tags\": \"\"}" >/dev/null + +rmq PUT "/permissions/$(urlencode "ofborg")/ofborg" \ + -d "{\"configure\":\".*\",\"write\":\".*\",\"read\":\".*\"}" >/dev/null + +echo "${pass}" > .ofborg-data/.amqp-password diff --git a/mprocs/start-rabbitmq.sh b/mprocs/start-rabbitmq.sh new file mode 100755 index 00000000..6161f256 --- /dev/null +++ b/mprocs/start-rabbitmq.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ADMIN_USER="${RABBITMQ_ADMIN_USER:-admin}" +ADMIN_PASS="${RABBITMQ_ADMIN_PASS:-admin}" +DATA_DIR="${DATA_DIR:-$(pwd)/.ofborg-data/rabbitmq}" +CONTAINER_NAME="${CONTAINER_NAME:-ofborg-rabbitmq}" + +mkdir -p "${DATA_DIR}" + +# Remove a stopped container with the same name if it exists +if docker inspect "${CONTAINER_NAME}" &>/dev/null; then + echo "Container '${CONTAINER_NAME}' already exists." + echo "Run 'docker rm -f ${CONTAINER_NAME}' to remove it first" + exit 1 +fi + +echo "Starting RabbitMQ..." +docker run \ + --name "${CONTAINER_NAME}" \ + --hostname ofborg-rabbitmq \ + --rm \ + -p 5672:5672 \ + -p 15672:15672 \ + -e RABBITMQ_DEFAULT_USER="${ADMIN_USER}" \ + -e RABBITMQ_DEFAULT_PASS="${ADMIN_PASS}" \ + -e RABBITMQ_DEFAULT_VHOST="/" \ + -v "${DATA_DIR}:/var/lib/rabbitmq" \ + rabbitmq:4-management diff --git a/mprocs/wait-for-rabbitmq.sh b/mprocs/wait-for-rabbitmq.sh new file mode 100755 index 00000000..e9d8679a --- /dev/null +++ b/mprocs/wait-for-rabbitmq.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail +set -x + +RABBIT_HOST=${RABBIT_HOST:-localhost} +RABBIT_USER=${RABBIT_USER:-admin} +RABBIT_PASS=${RABBIT_PASS:-admin} + +until curl -sf "http://${RABBIT_HOST}:15672/api/healthchecks/node" \ + -u "${RABBIT_USER}:${RABBIT_PASS}" &>/dev/null; do + echo "Waiting for RabbitMQ..." + sleep 1 +done + +echo "RabbitMQ is ready" diff --git a/ofborg-send-event/Cargo.toml b/ofborg-send-event/Cargo.toml new file mode 100644 index 00000000..f355b4b2 --- /dev/null +++ b/ofborg-send-event/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "ofborg-send-event" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1" +clap = { version = "4", features = ["derive"] } +hex = "0.4" +hmac = "0.13" +reqwest = { version = "0.13", features = ["json"] } +sha2 = "0.11" +tokio = { version = "1", features = ["full"] } +uuid = { version = "1", features = ["v4"] } +serde_json = "1.0.135" +tracing = "0.1.41" + +octocrab = { version = "0.51.0", default-features = false, features = [ + "default-client", + "follow-redirect", + "jwt-rust-crypto", + "retry", + "rustls", + # Enabling `rustls-ring` leads to runtime panics: https://github.com/XAMPPRocky/octocrab/issues/855 + # "rustls-ring", + "timeout", + "tracing", +] } +ofborg = { path = "../ofborg" } diff --git a/ofborg-send-event/src/main.rs b/ofborg-send-event/src/main.rs new file mode 100644 index 00000000..1d438055 --- /dev/null +++ b/ofborg-send-event/src/main.rs @@ -0,0 +1,208 @@ +use anyhow::{Context as _, Result, bail}; +use clap::Parser; +use hmac::KeyInit; +use hmac::{Hmac, Mac as _}; +use octocrab::Octocrab; +use ofborg::ghevent::{ + PullRequest, PullRequestAction, PullRequestEvent, PullRequestRef, PullRequestState, Repository, + User, +}; +use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue, USER_AGENT}; +use sha2::Sha256; +use std::str::FromStr as _; +use uuid::Uuid; + +#[derive(Debug, Parser)] +#[command(about = "Send a GitHub-style webhook to a local or staging receiver")] +struct Args { + /// Destination webhook URL + #[arg(long, default_value = "http://localhost:9899")] + webhook_receiver_url: String, + + /// PR that should be fetched + #[arg()] + pr_nr: u64, + + /// Webhook event name, e.g. push, pull_request, ping + #[arg(long, default_value = "pull_request")] + event: String, + + /// Shared secret path used to generate X-Hub-Signature-256 + #[arg(long)] + secret_path: Option, + + /// Delivery ID header. Defaults to a random UUID. + #[arg(long)] + delivery_id: Option, + + /// Add arbitrary header(s), format: Name=Value + #[arg(long = "header")] + headers: Vec, + + /// Print response headers too + #[arg(long)] + verbose: bool, + + /// Timeout in seconds + #[arg(long, default_value_t = 30)] + timeout_secs: u64, + + /// Full Repo Name which should be used as repo + #[arg(long, default_value = "NixOS/nixpkgs")] + full_repo_name: String, +} + +async fn make_pull_request_body(full_repo_name: &str, number: u64) -> Result { + let (org_name, repo_name) = full_repo_name + .split_once("/") + .with_context(|| format!("Unexpected Full Repo Name! {full_repo_name}"))?; + + let octocrab = Octocrab::builder() + .build() + .context("failed to create Octocrab client")?; + let pr = octocrab + .pulls(org_name, repo_name) + .get(number) + .await + .with_context(|| format!("failed to fetch PR {number}"))?; + tracing::info!("Fetched PR {number}: {}", pr.title); + + Ok(PullRequestEvent { + action: PullRequestAction::Opened, + number, + repository: Repository { + owner: User { + login: org_name.to_owned(), + }, + name: repo_name.to_owned(), + full_name: format!("{org_name}/{repo_name}"), + clone_url: format!("https://github.com/{org_name}/{repo_name}.git"), + }, + pull_request: PullRequest { + state: PullRequestState::Open, + base: PullRequestRef { + git_ref: pr.base.ref_field, + sha: pr.base.sha, + }, + head: PullRequestRef { + git_ref: pr.head.ref_field, + sha: pr.head.sha, + }, + }, + changes: None, + }) +} + +#[tokio::main] +async fn main() -> Result<()> { + ofborg::setup_log(); + let args = Args::parse(); + + let event = make_pull_request_body(&args.full_repo_name, args.pr_nr).await?; + let body = serde_json::to_vec(&event)?; + + let delivery_id = args + .delivery_id + .unwrap_or_else(|| Uuid::new_v4().to_string()); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(args.timeout_secs)) + .build() + .context("failed to build reqwest client")?; + + let mut headers = HeaderMap::new(); + headers.insert( + USER_AGENT, + HeaderValue::from_static("ofborg-send-event/0.1"), + ); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + HeaderName::from_static("x-github-event"), + HeaderValue::from_str(&args.event).context("invalid X-GitHub-Event value")?, + ); + headers.insert( + HeaderName::from_static("x-github-delivery"), + HeaderValue::from_str(&delivery_id).context("invalid X-GitHub-Delivery value")?, + ); + + if let Some(secret_path) = &args.secret_path { + let secret = std::fs::read_to_string(secret_path)?; + let signature = github_signature_256(secret.trim(), &body)?; + headers.insert( + HeaderName::from_static("x-hub-signature-256"), + HeaderValue::from_str(&signature).context("invalid X-Hub-Signature-256 value")?, + ); + } + + for raw in &args.headers { + let (name, value) = parse_header(raw)?; + headers.insert(name, value); + } + + let response = client + .post(&args.webhook_receiver_url) + .headers(headers) + .body(body) + .send() + .await + .with_context(|| format!("failed to POST to {}", args.webhook_receiver_url))?; + + let status = response.status(); + let resp_headers = response.headers().clone(); + let resp_body = response + .text() + .await + .context("failed to read response body")?; + + println!("status: {}", status); + + if args.verbose { + println!("response headers:"); + for (name, value) in resp_headers.iter() { + println!( + "{}: {}", + name.as_str(), + value.to_str().unwrap_or("") + ); + } + } + + if !resp_body.is_empty() { + println!(); + println!("{}", resp_body); + } + + Ok(()) +} + +fn github_verify_signature(secret: &str, body: &[u8], tag: &[u8]) -> Result { + let mut mac = + Hmac::::new_from_slice(secret.as_bytes()).context("invalid HMAC secret bytes")?; + mac.update(body); + Ok(mac.verify_slice(tag).is_ok()) +} + +fn github_signature_256(secret: &str, body: &[u8]) -> Result { + let mut mac = + Hmac::::new_from_slice(secret.as_bytes()).context("invalid HMAC secret bytes")?; + mac.update(body); + let tag = mac.finalize().into_bytes(); + assert!(github_verify_signature(secret, body, &tag)?); + Ok(format!("sha256={}", hex::encode(tag))) +} + +fn parse_header(input: &str) -> Result<(HeaderName, HeaderValue)> { + let Some((name, value)) = input.split_once('=') else { + bail!( + "invalid --header format: expected Name=Value, got {}", + input + ); + }; + + let name = HeaderName::from_str(name.trim()) + .with_context(|| format!("invalid header name: {}", name.trim()))?; + let value = HeaderValue::from_str(value.trim()) + .with_context(|| format!("invalid header value for {}", name))?; + + Ok((name, value)) +} diff --git a/ofborg-simple-build/Cargo.toml b/ofborg-simple-build/Cargo.toml index 107744e0..12d7067d 100644 --- a/ofborg-simple-build/Cargo.toml +++ b/ofborg-simple-build/Cargo.toml @@ -7,4 +7,4 @@ edition = "2024" [dependencies] ofborg = { path = "../ofborg" } -log = "0.4.25" +tracing = "0.1.41" diff --git a/ofborg-simple-build/src/main.rs b/ofborg-simple-build/src/main.rs index b27aa040..edc24596 100644 --- a/ofborg-simple-build/src/main.rs +++ b/ofborg-simple-build/src/main.rs @@ -1,5 +1,3 @@ -extern crate log; - use std::env; use std::fs::File; use std::io::Read; @@ -11,11 +9,11 @@ use ofborg::nix; fn main() { ofborg::setup_log(); - log::info!("Loading config..."); + tracing::info!("Loading config..."); let cfg = config::load(env::args().nth(1).unwrap().as_ref()); let nix = cfg.nix(); - log::info!("Running build..."); + tracing::info!("Running build..."); match nix.safely_build_attrs( Path::new("./"), nix::File::DefaultNixpkgs, diff --git a/ofborg/Cargo.toml b/ofborg/Cargo.toml index 04c6ad34..46e7340d 100644 --- a/ofborg/Cargo.toml +++ b/ofborg/Cargo.toml @@ -6,39 +6,51 @@ build = "build.rs" edition = "2024" [dependencies] +anyhow = "1.0.0" async-trait = "0.1.89" brace-expand = "0.1.0" +bytes = "1.11.1" chrono = { version = "0.4.38", default-features = false, features = [ "clock", "std", ] } -either = "1.13.0" fs2 = "0.4.3" futures = "0.3.31" -futures-util = "0.3.31" hex = "0.4.3" hmac = "0.13.0" http = "1" +http-body = "1" http-body-util = "0.1" -#hubcaps = "0.6" -# for Conclusion::Skipped which is in master -hubcaps = { git = "https://github.com/ofborg/hubcaps.git", rev = "0d7466ef941a7a8e160c071e2846e56b90b6ea86" } +octocrab = { version = "0.51.0", default-features = false, features = [ + "default-client", + "follow-redirect", + "jwt-rust-crypto", + "retry", + "rustls", + # Enabling `rustls-ring` leads to runtime panics: https://github.com/XAMPPRocky/octocrab/issues/855 + # "rustls-ring", + "timeout", + "tracing", +] } +jsonwebtoken = "10" hyper = { version = "1.0", features = ["full", "server", "http1"] } hyper-util = { version = "0.1", features = ["server", "tokio", "http1"] } lapin = "4.3.0" lru-cache = "0.1.2" md5 = "0.8.0" -mime = "0.3" nom = "8" parking_lot = "0.12.4" regex = "1.11.1" -rustls-pki-types = "1.14" serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.135" sha2 = "0.11.0" tempfile = "3.15.0" -tokio = { version = "1", features = ["rt-multi-thread", "net", "macros", "sync"] } +tokio = { version = "1", features = ["full"] } tokio-stream = "0.1" tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", features = ["json", "env-filter"] } uuid = { version = "1.12", features = ["v4"] } + +[dev-dependencies] +bytes = "1.11.1" +http-body = "1.0.1" diff --git a/ofborg/src/acl.rs b/ofborg/src/acl.rs index 805fbb3b..cda86fde 100644 --- a/ofborg/src/acl.rs +++ b/ofborg/src/acl.rs @@ -35,17 +35,6 @@ impl Acl { } } - pub fn build_job_destinations_for_user_repo( - &self, - user: &str, - repo: &str, - ) -> Vec<(Option, Option)> { - self.build_job_architectures_for_user_repo(user, repo) - .iter() - .map(|system| system.as_build_destination()) - .collect() - } - pub fn can_build_unrestricted(&self, user: &str, repo: &str) -> bool { if let Some(ref users) = self.trusted_users { if repo.to_lowercase() == "nixos/nixpkgs" { diff --git a/ofborg/src/bin/build-faker.rs b/ofborg/src/bin/build-faker.rs deleted file mode 100644 index e6f1cd25..00000000 --- a/ofborg/src/bin/build-faker.rs +++ /dev/null @@ -1,62 +0,0 @@ -use lapin::message::Delivery; -use std::env; -use std::error::Error; - -use ofborg::commentparser; -use ofborg::config; -use ofborg::easylapin; -use ofborg::message::{Pr, Repo, buildjob}; -use ofborg::notifyworker::NotificationReceiver; -use ofborg::worker; - -#[tokio::main] -async fn main() -> Result<(), Box> { - ofborg::setup_log(); - - let arg = env::args().nth(1).expect("usage: build-faker "); - let cfg = config::load(arg.as_ref()); - - let conn = easylapin::from_config(&cfg.builder.unwrap().rabbitmq).await?; - let chan = conn.create_channel().await?; - - let repo_msg = Repo { - clone_url: "https://github.com/nixos/ofborg.git".to_owned(), - full_name: "NixOS/ofborg".to_owned(), - owner: "NixOS".to_owned(), - name: "ofborg".to_owned(), - }; - - let pr_msg = Pr { - number: 42, - head_sha: "6dd9f0265d52b946dd13daf996f30b64e4edb446".to_owned(), - target_branch: Some("scratch".to_owned()), - }; - - let logbackrk = "NixOS/ofborg.42".to_owned(); - - let msg = buildjob::BuildJob { - repo: repo_msg, - pr: pr_msg, - subset: Some(commentparser::Subset::Nixpkgs), - attrs: vec!["success".to_owned()], - logs: Some((Some("logs".to_owned()), Some(logbackrk.to_lowercase()))), - statusreport: Some((None, Some("scratch".to_owned()))), - request_id: "bogus-request-id".to_owned(), - }; - - { - let deliver = Delivery::mock(0, "no-exchange".into(), "".into(), false, vec![]); - let recv = easylapin::ChannelNotificationReceiver::new(chan.clone(), deliver); - - for _i in 1..2 { - recv.tell(worker::publish_serde_action( - None, - Some("build-inputs-x86_64-darwin".to_owned()), - &msg, - )) - .await; - } - } - - Ok(()) -} diff --git a/ofborg/src/bin/builder.rs b/ofborg/src/bin/builder.rs deleted file mode 100644 index 85c7b1e2..00000000 --- a/ofborg/src/bin/builder.rs +++ /dev/null @@ -1,116 +0,0 @@ -use std::env; -use std::error::Error; -use std::future::Future; -use std::path::Path; -use std::pin::Pin; - -use futures_util::future; -use tracing::{error, info, warn}; - -use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::{checkout, config, tasks}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - ofborg::setup_log(); - - let arg = env::args() - .nth(1) - .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); - let cfg = config::load(arg.as_ref()); - - let Some(builder_cfg) = config::load(arg.as_ref()).builder else { - error!("No builder configuration found!"); - panic!(); - }; - - let conn = easylapin::from_config(&builder_cfg.rabbitmq).await?; - let mut handles: Vec + Send>>> = Vec::new(); - - for system in &cfg.nix.system { - handles.push(self::create_handle(&conn, &cfg, system.to_string()).await?); - } - - future::join_all(handles).await; - - drop(conn); // Close connection. - info!("Closed the session... EOF"); - Ok(()) -} - -#[allow(clippy::type_complexity)] -async fn create_handle( - conn: &lapin::Connection, - cfg: &config::Config, - system: String, -) -> Result + Send>>, Box> { - let mut chan = conn.create_channel().await?; - - let cloner = checkout::cached_cloner(Path::new(&cfg.checkout.root)); - let nix = cfg.nix().with_system(system.clone()); - - chan.declare_exchange(easyamqp::ExchangeConfig { - exchange: "build-jobs".to_owned(), - exchange_type: easyamqp::ExchangeType::Fanout, - passive: false, - durable: true, - auto_delete: false, - no_wait: false, - internal: false, - }) - .await?; - - let queue_name = if cfg.runner.build_all_jobs != Some(true) { - let queue_name = format!("build-inputs-{system}"); - chan.declare_queue(easyamqp::QueueConfig { - queue: queue_name.clone(), - passive: false, - durable: true, - exclusive: false, - auto_delete: false, - no_wait: false, - }) - .await?; - queue_name - } else { - warn!("Building all jobs, please don't use this unless you're"); - warn!("developing and have Graham's permission!"); - let queue_name = "".to_owned(); - chan.declare_queue(easyamqp::QueueConfig { - queue: queue_name.clone(), - passive: false, - durable: false, - exclusive: true, - auto_delete: true, - no_wait: false, - }) - .await?; - queue_name - }; - - chan.bind_queue(easyamqp::BindQueueConfig { - queue: queue_name.clone(), - exchange: "build-jobs".to_owned(), - routing_key: None, - no_wait: false, - }) - .await?; - - let handle = easylapin::NotifyChannel(chan) - .consume( - tasks::build::BuildWorker::new(cloner, nix, system, cfg.runner.identity.clone()), - easyamqp::ConsumeConfig { - queue: queue_name.clone(), - consumer_tag: format!("{}-builder", cfg.whoami()), - no_local: false, - no_ack: false, - no_wait: false, - exclusive: false, - }, - ) - .await?; - - info!("Fetching jobs from {}", &queue_name); - Ok(handle) -} diff --git a/ofborg/src/bin/evaluation-filter.rs b/ofborg/src/bin/evaluation-filter.rs index eda925f9..edba6848 100644 --- a/ofborg/src/bin/evaluation-filter.rs +++ b/ofborg/src/bin/evaluation-filter.rs @@ -1,18 +1,13 @@ -use std::env; -use std::error::Error; - use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); diff --git a/ofborg/src/bin/github-comment-filter.rs b/ofborg/src/bin/github-comment-filter.rs index 4d1427e0..3fca4424 100644 --- a/ofborg/src/bin/github-comment-filter.rs +++ b/ofborg/src/bin/github-comment-filter.rs @@ -1,19 +1,14 @@ -use std::env; -use std::error::Error; - use ofborg::systems::System; use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); diff --git a/ofborg/src/bin/github-comment-poster.rs b/ofborg/src/bin/github-comment-poster.rs index 3b26b83c..03e2d630 100644 --- a/ofborg/src/bin/github-comment-poster.rs +++ b/ofborg/src/bin/github-comment-poster.rs @@ -1,18 +1,13 @@ -use std::env; -use std::error::Error; - use tracing::{error, info}; -use ofborg::config; use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; +use ofborg::{config, easylapin, tasks}; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); @@ -56,7 +51,9 @@ async fn main() -> Result<(), Box> { let handle = easylapin::WorkerChannel(chan) .consume( - tasks::githubcommentposter::GitHubCommentPoster::new(cfg.github_app_vendingmachine()), + tasks::githubcommentposter::GitHubCommentPoster::new( + cfg.github_app_vendingmachine().unwrap(), + ), easyamqp::ConsumeConfig { queue: "build-results".to_owned(), consumer_tag: format!("{}-github-comment-poster", cfg.whoami()), diff --git a/ofborg/src/bin/github-webhook-receiver.rs b/ofborg/src/bin/github-webhook-receiver.rs index f31ac793..9cc2452f 100644 --- a/ofborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/src/bin/github-webhook-receiver.rs @@ -1,28 +1,55 @@ -use std::env; -use std::error::Error; use std::net::SocketAddr; use std::sync::Arc; -use hmac::{Hmac, KeyInit as _, Mac}; +use async_trait::async_trait; +use hmac::{Hmac, KeyInit as _, Mac as _}; use http::{Method, StatusCode}; -use http_body_util::{BodyExt, Full}; -use hyper::body::{Bytes, Incoming}; +use http_body_util::{BodyExt as _, Full}; +use hyper::body::Bytes; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper::{Request, Response}; use hyper_util::rt::TokioIo; -use lapin::options::BasicPublishOptions; -use lapin::{BasicProperties, Channel}; -use ofborg::ghevent::GenericWebhook; -use ofborg::{config, easyamqp, easyamqp::ChannelExt, easylapin}; use sha2::Sha256; use tokio::net::TcpListener; use tokio::sync::Mutex; use tracing::{error, info, warn}; +use ofborg::ghevent::GenericWebhook; +use ofborg::{MessagePublisher, config, easyamqp, easyamqp::ChannelExt, easylapin}; + +pub struct LapinPublisher { + chan: Arc>, +} + +impl LapinPublisher { + pub fn new(chan: Arc>) -> Self { + Self { chan } + } +} + +#[async_trait] +impl MessagePublisher for LapinPublisher { + async fn publish(&self, exchange: &str, routing_key: &str, body: &[u8]) -> anyhow::Result<()> { + let chan = self.chan.lock().await; + let _confirmation = chan + .basic_publish( + exchange.into(), + routing_key.into(), + lapin::options::BasicPublishOptions::default(), + body, + lapin::BasicProperties::default() + .with_content_type("application/json".into()) + .with_delivery_mode(2), + ) + .await?; + Ok(()) + } +} + /// Prepares the the exchange we will write to, the queues that are bound to it /// and binds them. -async fn setup_amqp(chan: &mut Channel) -> Result<(), Box> { +async fn setup_amqp(chan: &mut lapin::Channel) -> anyhow::Result<()> { chan.declare_exchange(easyamqp::ExchangeConfig { exchange: "github-events".to_owned(), exchange_type: easyamqp::ExchangeType::Topic, @@ -104,11 +131,16 @@ fn empty_response(status: StatusCode) -> Response> { .unwrap() } -async fn handle_request( - req: Request, +async fn handle_request( + req: Request, webhook_secret: Arc, - chan: Arc>, -) -> Result>, hyper::Error> { + publisher: Arc, +) -> Result>, hyper::Error> +where + B: http_body::Body + Send + Sync, + F: bytes::Buf, + B::Error: std::fmt::Debug + Send + Sync, +{ // HTTP 405 if req.method() != Method::POST { return Ok(empty_response(StatusCode::METHOD_NOT_ALLOWED)); @@ -135,7 +167,7 @@ async fn handle_request( let raw = match req.collect().await { Ok(collected) => collected.to_bytes(), Err(e) => { - warn!("Failed to read body from client: {e}"); + warn!("Failed to read body from client: {e:?}"); return Ok(response( StatusCode::INTERNAL_SERVER_ERROR, "Failed to read body", @@ -215,30 +247,33 @@ async fn handle_request( let Some(event_type) = event_type else { return Ok(response(StatusCode::BAD_REQUEST, "Missing event type")); }; - let routing_key = format!("{event_type}.{}", input.repository.full_name.to_lowercase()); + let routing_key = match event_type.as_str() { + "pull_request" | "issue_comment" => { + format!("{event_type}.{}", input.repository.full_name.to_lowercase()) + } + other => { + warn!("Received unknown event type: {other}"); + format!("unknown.{other}") + } + }; // Publish message - let chan = chan.lock().await; - let _confirmation = chan - .basic_publish( - "github-events".into(), - routing_key.as_str().into(), - BasicPublishOptions::default(), - &raw, - BasicProperties::default() - .with_content_type("application/json".into()) - .with_delivery_mode(2), // persistent - ) - .await; + if let Err(e) = publisher.publish("github-events", &routing_key, &raw).await { + error!("Failed to publish message: {e}"); + return Ok(response( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to publish message", + )); + } Ok(empty_response(StatusCode::NO_CONTENT)) } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let Some(cfg) = config::load(arg.as_ref()).github_webhook_receiver else { @@ -254,6 +289,7 @@ async fn main() -> Result<(), Box> { let mut chan = conn.create_channel().await?; setup_amqp(&mut chan).await?; let chan = Arc::new(Mutex::new(chan)); + let publisher: Arc = Arc::new(LapinPublisher::new(chan.clone())); let addr: SocketAddr = cfg.listen.parse()?; let listener = TcpListener::bind(addr).await?; @@ -264,11 +300,12 @@ async fn main() -> Result<(), Box> { let io = TokioIo::new(stream); let webhook_secret = webhook_secret.clone(); - let chan = chan.clone(); + let publisher = publisher.clone(); tokio::task::spawn(async move { - let service = - service_fn(move |req| handle_request(req, webhook_secret.clone(), chan.clone())); + let service = service_fn(move |req| { + handle_request(req, webhook_secret.clone(), publisher.clone()) + }); if let Err(err) = http1::Builder::new().serve_connection(io, service).await { warn!("Error serving connection: {:?}", err); @@ -276,3 +313,325 @@ async fn main() -> Result<(), Box> { }); } } + +mod github_webhook_receiver { + #[cfg(test)] + mod test { + use super::super::*; + use http::header; + + use hyper::body::Bytes; + use ofborg::test_utils::MockPublisher; + + fn create_request( + method: http::Method, + headers: Vec<(http::header::HeaderName, http::header::HeaderValue)>, + body: Bytes, + ) -> http::Request> { + let mut builder = http::Request::builder().method(method); + for (name, value) in headers { + builder = builder.header(name, value); + } + let body = http_body_util::Full::new(body); + builder.body(body).unwrap() + } + + fn hv(s: &str) -> http::header::HeaderValue { + http::header::HeaderValue::from_str(s).unwrap() + } + + fn hn(s: &'static str) -> http::header::HeaderName { + http::header::HeaderName::from_bytes(s.as_bytes()).unwrap() + } + + fn compute_signature(secret: &str, body: &[u8]) -> String { + let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); + mac.update(body); + let result = mac.finalize(); + format!("sha256={}", hex::encode(result.into_bytes())) + } + + fn valid_headers( + secret: &str, + body: &[u8], + ) -> Vec<(header::HeaderName, header::HeaderValue)> { + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature(secret, body)), + ), + (hn("X-Github-Event"), hv("pull_request")), + ] + } + + fn minimal_valid_webhook() -> &'static str { + r#"{"repository":{"owner":{"login":"test"},"name":"test-repo","full_name":"test/test-repo","clone_url":"https://github.com/test/test-repo.git"}}"# + } + + fn pr_event_webhook() -> &'static str { + include_str!("../../test-srcs/events/pr-changed-base.json") + } + + #[tokio::test] + async fn test_method_not_allowed() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let req = create_request(http::Method::GET, vec![], Bytes::new()); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); + } + + #[tokio::test] + async fn test_missing_signature_header() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![(header::CONTENT_TYPE, hv("application/json"))], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_invalid_signature_hash_method() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + (hn("X-Hub-Signature-256"), hv("sha1=abc123")), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_signature_verification_failed() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv( + "sha256=0000000000000000000000000000000000000000000000000000000000000000", + ), + ), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_missing_content_type() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![( + hn("X-Hub-Signature-256"), + hv(&compute_signature( + "test-secret", + minimal_valid_webhook().as_bytes(), + )), + )], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_invalid_content_type() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("text/plain")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature( + "test-secret", + minimal_valid_webhook().as_bytes(), + )), + ), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_invalid_json() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from("not valid json {{{"); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", b"not valid json {{{")), + ), + (hn("X-Github-Event"), hv("pull_request")), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_missing_event_type() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = Bytes::from(minimal_valid_webhook()); + let req = create_request( + http::Method::POST, + vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature( + "test-secret", + minimal_valid_webhook().as_bytes(), + )), + ), + ], + body, + ); + + let resp = handle_request(req, secret, publisher).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_successful_webhook_with_routing_key() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body_bytes = pr_event_webhook().as_bytes(); + let req = create_request( + http::Method::POST, + valid_headers("test-secret", body_bytes), + Bytes::from(pr_event_webhook()), + ); + + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published.len(), 1); + assert_eq!(published[0].exchange, "github-events"); + assert_eq!(published[0].routing_key, "pull_request.nixos/nixpkgs"); + assert_eq!(&published[0].body, pr_event_webhook().as_bytes()); + } + + #[tokio::test] + async fn test_issue_comment_routing_key() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = r#"{"repository":{"owner":{"login":"test"},"name":"my-repo","full_name":"test/my-repo","clone_url":"https://github.com/test/my-repo.git"}}"#; + let body_bytes = body.as_bytes(); + + let headers = vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", body_bytes)), + ), + (hn("X-Github-Event"), hv("issue_comment")), + ]; + + let req = create_request(http::Method::POST, headers, Bytes::from(body)); + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published[0].routing_key, "issue_comment.test/my-repo"); + } + + #[tokio::test] + async fn test_routing_key_lowercases_repo_name() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = r#"{"repository":{"owner":{"login":"Test"},"name":"MyRepo","full_name":"Test/MyRepo","clone_url":"https://github.com/Test/MyRepo.git"}}"#; + let body_bytes = body.as_bytes(); + + let headers = vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", body_bytes)), + ), + (hn("X-Github-Event"), hv("pull_request")), + ]; + + let req = create_request(http::Method::POST, headers, Bytes::from(body)); + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published[0].routing_key, "pull_request.test/myrepo"); + } + + #[tokio::test] + async fn test_unknown_event_type_routing_key() { + let publisher = Arc::new(MockPublisher::new()); + let secret = Arc::new("test-secret".to_string()); + let body = r#"{"repository":{"owner":{"login":"test"},"name":"my-repo","full_name":"test/my-repo","clone_url":"https://github.com/test/my-repo.git"}}"#; + let body_bytes = body.as_bytes(); + + let headers = vec![ + (header::CONTENT_TYPE, hv("application/json")), + ( + hn("X-Hub-Signature-256"), + hv(&compute_signature("test-secret", body_bytes)), + ), + (hn("X-Github-Event"), hv("push")), + ]; + + let req = create_request(http::Method::POST, headers, Bytes::from(body)); + let resp = handle_request(req, secret, publisher.clone()) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let published = publisher.get_published().await; + assert_eq!(published[0].routing_key, "unknown.push"); + } + } +} diff --git a/ofborg/src/bin/log-message-collector.rs b/ofborg/src/bin/log-message-collector.rs deleted file mode 100644 index 3ad668cb..00000000 --- a/ofborg/src/bin/log-message-collector.rs +++ /dev/null @@ -1,83 +0,0 @@ -use std::env; -use std::error::Error; -use std::path::PathBuf; - -use tracing::{error, info}; - -use ofborg::config; -use ofborg::easyamqp::{self, ChannelExt, ConsumerExt}; -use ofborg::easylapin; -use ofborg::tasks; - -#[tokio::main] -async fn main() -> Result<(), Box> { - ofborg::setup_log(); - - let arg = env::args() - .nth(1) - .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); - let cfg = config::load(arg.as_ref()); - - let Some(collector_cfg) = config::load(arg.as_ref()).log_message_collector else { - error!("No log message collector configuration found!"); - panic!(); - }; - - let conn = easylapin::from_config(&collector_cfg.rabbitmq).await?; - let mut chan = conn.create_channel().await?; - - chan.declare_exchange(easyamqp::ExchangeConfig { - exchange: "logs".to_owned(), - exchange_type: easyamqp::ExchangeType::Topic, - passive: false, - durable: true, - auto_delete: false, - no_wait: false, - internal: false, - }) - .await?; - - let queue_name = "logs".to_owned(); - chan.declare_queue(easyamqp::QueueConfig { - queue: queue_name.clone(), - passive: false, - durable: false, - exclusive: true, - auto_delete: true, - no_wait: false, - }) - .await?; - - chan.bind_queue(easyamqp::BindQueueConfig { - queue: queue_name.clone(), - exchange: "logs".to_owned(), - routing_key: Some("*.*".to_owned()), - no_wait: false, - }) - .await?; - - // Regular channel, we want prefetching here. - let handle = chan - .consume( - tasks::log_message_collector::LogMessageCollector::new( - PathBuf::from(collector_cfg.logs_path), - 100, - ), - easyamqp::ConsumeConfig { - queue: queue_name.clone(), - consumer_tag: format!("{}-log-collector", cfg.whoami()), - no_local: false, - no_ack: false, - no_wait: false, - exclusive: false, - }, - ) - .await?; - - info!("Fetching jobs from {}", &queue_name); - handle.await; - - drop(conn); // Close connection. - info!("Closed the session... EOF"); - Ok(()) -} diff --git a/ofborg/src/bin/logapi.rs b/ofborg/src/bin/logapi.rs deleted file mode 100644 index b47e1acf..00000000 --- a/ofborg/src/bin/logapi.rs +++ /dev/null @@ -1,157 +0,0 @@ -use std::net::SocketAddr; -use std::{collections::HashMap, error::Error, path::PathBuf, sync::Arc}; - -use http::{Method, StatusCode}; -use http_body_util::Full; -use hyper::body::Bytes; -use hyper::server::conn::http1; -use hyper::service::service_fn; -use hyper::{Request, Response}; -use hyper_util::rt::TokioIo; -use ofborg::config; -use tokio::net::TcpListener; -use tracing::{error, info, warn}; - -#[derive(serde::Serialize, Default)] -struct Attempt { - metadata: Option, - result: Option, - log_url: Option, -} - -#[derive(serde::Serialize)] -struct LogResponse { - attempts: HashMap, -} - -#[derive(Clone)] -struct LogApiConfig { - logs_path: PathBuf, - serve_root: String, -} - -fn response(status: StatusCode, body: &'static str) -> Response> { - Response::builder() - .status(status) - .body(Full::new(Bytes::from(body))) - .unwrap() -} - -fn json_response(status: StatusCode, body: String) -> Response> { - Response::builder() - .status(status) - .header("Content-Type", "application/json") - .body(Full::new(Bytes::from(body))) - .unwrap() -} - -async fn handle_request( - req: Request, - cfg: Arc, -) -> Result>, hyper::Error> { - if req.method() != Method::GET { - return Ok(response(StatusCode::METHOD_NOT_ALLOWED, "")); - } - - let uri = req.uri().path().to_string(); - let Some(reqd) = uri.strip_prefix("/logs/").map(ToOwned::to_owned) else { - return Ok(response(StatusCode::NOT_FOUND, "invalid uri")); - }; - let path: PathBuf = cfg.logs_path.join(&reqd); - let Ok(path) = std::fs::canonicalize(&path) else { - return Ok(response(StatusCode::NOT_FOUND, "absent")); - }; - if !path.starts_with(&cfg.logs_path) { - return Ok(response(StatusCode::NOT_FOUND, "invalid path")); - } - let Ok(iter) = std::fs::read_dir(path) else { - return Ok(response(StatusCode::NOT_FOUND, "non dir")); - }; - - let mut attempts = HashMap::::new(); - for e in iter { - let Ok(e) = e else { continue }; - let e_metadata = e.metadata(); - if e_metadata.as_ref().map(|v| v.is_dir()).unwrap_or(true) { - return Ok(response(StatusCode::INTERNAL_SERVER_ERROR, "dir found")); - } - - if e_metadata.as_ref().map(|v| v.is_file()).unwrap_or_default() { - let Ok(file_name) = e.file_name().into_string() else { - warn!("entry filename is not a utf-8 string: {:?}", e.file_name()); - continue; - }; - - if file_name.ends_with(".metadata.json") || file_name.ends_with(".result.json") { - let Ok(file) = std::fs::File::open(e.path()) else { - warn!("could not open file: {file_name}"); - continue; - }; - let Ok(json) = serde_json::from_reader::<_, serde_json::Value>(file) else { - warn!("file is not a valid json file: {file_name}"); - continue; - }; - let Some(attempt_id) = json - .get("attempt_id") - .and_then(|v| v.as_str()) - .map(ToOwned::to_owned) - else { - warn!("attempt_id not found in file: {file_name}"); - continue; - }; - let attempt_obj = attempts.entry(attempt_id).or_default(); - if file_name.ends_with(".metadata.json") { - attempt_obj.metadata = Some(json); - } else { - attempt_obj.result = Some(json); - } - } else { - let attempt_obj = attempts.entry(file_name.clone()).or_default(); - attempt_obj.log_url = Some(format!("{}/{reqd}/{file_name}", &cfg.serve_root)); - } - } - } - - let body = serde_json::to_string(&LogResponse { attempts }).unwrap_or_default(); - Ok(json_response(StatusCode::OK, body)) -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - ofborg::setup_log(); - - let arg = std::env::args() - .nth(1) - .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); - let Some(cfg) = config::load(arg.as_ref()).log_api_config else { - error!("No LogApi configuration found!"); - panic!(); - }; - - let logs_path = std::fs::canonicalize(&cfg.logs_path) - .expect("logs_path does not exist or is not accessible"); - - let api_cfg = Arc::new(LogApiConfig { - logs_path, - serve_root: cfg.serve_root, - }); - - let addr: SocketAddr = cfg.listen.parse()?; - let listener = TcpListener::bind(addr).await?; - info!("Listening on {}", addr); - - loop { - let (stream, _) = listener.accept().await?; - let io = TokioIo::new(stream); - - let api_cfg = api_cfg.clone(); - - tokio::task::spawn(async move { - let service = service_fn(move |req| handle_request(req, api_cfg.clone())); - - if let Err(err) = http1::Builder::new().serve_connection(io, service).await { - warn!("Error serving connection: {:?}", err); - } - }); - } -} diff --git a/ofborg/src/bin/mass-rebuilder.rs b/ofborg/src/bin/mass-rebuilder.rs index 10877590..6507cfbf 100644 --- a/ofborg/src/bin/mass-rebuilder.rs +++ b/ofborg/src/bin/mass-rebuilder.rs @@ -1,5 +1,3 @@ -use std::env; -use std::error::Error; use std::path::Path; use tracing::{error, info}; @@ -12,10 +10,10 @@ use ofborg::stats; use ofborg::tasks; #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); @@ -44,14 +42,37 @@ async fn main() -> Result<(), Box> { }) .await?; + let hydra_eval_cfg = cfg.hydra_evaluator.clone(); + let (hydra_eval_queue, hydra_eval_nix, hydra_eval_jobset_id) = + if let Some(ref hec) = hydra_eval_cfg { + chan.declare_queue(easyamqp::QueueConfig { + queue: String::from("hydra-eval-jobs"), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + }) + .await?; + ( + Some("hydra-eval-jobs".to_owned()), + Some(cfg.nix()), + Some(hec.jobset_id), + ) + } else { + (None, None, None) + }; + let handle = easylapin::WorkerChannel(chan) .consume( tasks::evaluate::EvaluationWorker::new( cloner, cfg.github_app_vendingmachine(), - cfg.acl(), cfg.runner.identity.clone(), events, + hydra_eval_queue, + hydra_eval_nix, + hydra_eval_jobset_id, ), easyamqp::ConsumeConfig { queue: queue_name.clone(), diff --git a/ofborg/src/bin/stats.rs b/ofborg/src/bin/stats.rs index 72b04e97..3bd345f9 100644 --- a/ofborg/src/bin/stats.rs +++ b/ofborg/src/bin/stats.rs @@ -1,9 +1,8 @@ -use std::env; -use std::error::Error; use std::net::SocketAddr; use std::sync::Arc; -use http::StatusCode; +use http::header::CONTENT_TYPE; +use http::{HeaderValue, StatusCode}; use http_body_util::Full; use hyper::body::Bytes; use hyper::server::conn::http1; @@ -19,6 +18,10 @@ use ofborg::{config, easyamqp, easylapin, stats, tasks}; fn response(body: String) -> Response> { Response::builder() .status(StatusCode::OK) + .header( + CONTENT_TYPE, + HeaderValue::from_static("text/plain; version=0.0.4; charset=utf-8"), + ) .body(Full::new(Bytes::from(body))) .unwrap() } @@ -26,7 +29,7 @@ fn response(body: String) -> Response> { async fn run_http_server( addr: SocketAddr, metrics: Arc, -) -> Result<(), Box> { +) -> anyhow::Result<()> { let listener = TcpListener::bind(addr).await?; info!("HTTP server listening on {}", addr); @@ -50,10 +53,10 @@ async fn run_http_server( } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> anyhow::Result<()> { ofborg::setup_log(); - let arg = env::args() + let arg = std::env::args() .nth(1) .unwrap_or_else(|| panic!("usage: {} ", std::env::args().next().unwrap())); let cfg = config::load(arg.as_ref()); @@ -62,6 +65,7 @@ async fn main() -> Result<(), Box> { error!("No stats configuration found!"); panic!(); }; + let addr: SocketAddr = stats_cfg.listen.parse().unwrap(); let conn = easylapin::from_config(&stats_cfg.rabbitmq).await?; @@ -118,8 +122,7 @@ async fn main() -> Result<(), Box> { // Spawn HTTP server in a separate thread with its own tokio runtime let metrics_clone = metrics.clone(); - std::thread::spawn(async move || { - let addr: SocketAddr = "0.0.0.0:9898".parse().unwrap(); + tokio::task::spawn(async move { if let Err(e) = run_http_server(addr, metrics_clone).await { error!("HTTP server error: {:?}", e); } diff --git a/ofborg/src/checkout.rs b/ofborg/src/checkout.rs index 731e68d3..2c489da8 100644 --- a/ofborg/src/checkout.rs +++ b/ofborg/src/checkout.rs @@ -1,5 +1,3 @@ -use crate::clone::{self, GitClonable}; - use std::ffi::{OsStr, OsString}; use std::fs; use std::io::Error; @@ -8,6 +6,8 @@ use std::process::{Command, Stdio}; use tracing::info; +use crate::clone::{self, GitClonable}; + pub struct CachedCloner { root: PathBuf, } diff --git a/ofborg/src/clone.rs b/ofborg/src/clone.rs index 0dcb71c2..c678b0a5 100644 --- a/ofborg/src/clone.rs +++ b/ofborg/src/clone.rs @@ -1,11 +1,10 @@ -use fs2::FileExt; - use std::ffi::OsStr; use std::fs; use std::io::Error; use std::path::PathBuf; use std::process::{Command, Stdio}; +use fs2::FileExt as _; use tracing::{debug, info, warn}; pub struct Lock { diff --git a/ofborg/src/commitstatus.rs b/ofborg/src/commitstatus.rs index 6747f3b0..bcc86216 100644 --- a/ofborg/src/commitstatus.rs +++ b/ofborg/src/commitstatus.rs @@ -1,33 +1,37 @@ -use futures_util::future::TryFutureExt; +use octocrab::{self, models::StatusState}; use tracing::warn; +use crate::github::GithubRepo; + pub struct CommitStatus { - api: hubcaps::statuses::Statuses, + repo: GithubRepo, sha: String, context: String, description: String, url: String, + enable_publish: bool, } impl CommitStatus { pub fn new( - api: hubcaps::statuses::Statuses, + repo: GithubRepo, sha: String, context: String, description: String, url: Option, ) -> CommitStatus { - let mut stat = CommitStatus { - api, + CommitStatus { + repo, sha, context, description, - url: "".to_owned(), - }; - - stat.set_url(url); + url: url.unwrap_or_else(|| String::from("")), + enable_publish: true, + } + } - stat + pub fn set_enable_publish(&mut self, enable_publish: bool) { + self.enable_publish = enable_publish; } pub fn set_url(&mut self, url: Option) { @@ -37,7 +41,7 @@ impl CommitStatus { pub async fn set_with_description( &mut self, description: &str, - state: hubcaps::statuses::State, + state: StatusState, ) -> Result<(), CommitStatusError> { self.set_description(description.to_owned()); self.set(state).await @@ -47,7 +51,11 @@ impl CommitStatus { self.description = description; } - pub async fn set(&self, state: hubcaps::statuses::State) -> Result<(), CommitStatusError> { + pub async fn set(&self, state: StatusState) -> Result<(), CommitStatusError> { + if !self.enable_publish { + return Ok(()); + } + let desc = if self.description.len() >= 140 { warn!( "description is over 140 char; truncating: {:?}", @@ -57,47 +65,28 @@ impl CommitStatus { } else { self.description.clone() }; - self.api - .create( - self.sha.as_ref(), - &hubcaps::statuses::StatusOptions::builder(state) - .context(self.context.clone()) - .description(desc) - .target_url(self.url.clone()) - .build(), - ) - .map_ok(|_| ()) - .map_err(|e| CommitStatusError::from(e)) + + self.repo + .repos() + .create_status(self.sha.clone(), state) + .context(self.context.clone()) + .description(desc) + .target(self.url.clone()) + .send() .await?; + Ok(()) } } #[derive(Debug)] pub enum CommitStatusError { - ExpiredCreds(hubcaps::Error), - MissingSha(hubcaps::Error), - Error(hubcaps::Error), + OctocrabError(octocrab::Error), InternalError(String), } -impl From for CommitStatusError { - fn from(e: hubcaps::Error) -> CommitStatusError { - use http::status::StatusCode; - use hubcaps::Error; - match &e { - Error::Fault { code, error } - if code == &StatusCode::UNAUTHORIZED && error.message == "Bad credentials" => - { - CommitStatusError::ExpiredCreds(e) - } - Error::Fault { code, error } - if code == &StatusCode::UNPROCESSABLE_ENTITY - && error.message.starts_with("No commit found for SHA:") => - { - CommitStatusError::MissingSha(e) - } - _otherwise => CommitStatusError::Error(e), - } +impl From for CommitStatusError { + fn from(e: octocrab::Error) -> CommitStatusError { + CommitStatusError::OctocrabError(e) } } diff --git a/ofborg/src/config.rs b/ofborg/src/config.rs index 542eeedd..28cde14a 100644 --- a/ofborg/src/config.rs +++ b/ofborg/src/config.rs @@ -1,18 +1,18 @@ -use crate::acl; -use crate::nix::Nix; - -use std::collections::{HashMap, hash_map::Entry}; +use std::collections::HashMap; use std::fmt; use std::fs::File; -use std::io::Read; +use std::io::Read as _; use std::marker::PhantomData; use std::path::{Path, PathBuf}; -use hubcaps::{Credentials, Github, InstallationTokenGenerator, JWTCredentials}; -use rustls_pki_types::pem::PemObject as _; +use octocrab::models::InstallationId; +use octocrab::{Octocrab, auth::AppAuth}; use serde::de::{self, Deserializer}; use tracing::{debug, error, info, warn}; +use crate::acl; +use crate::nix::Nix; + /// Main ofBorg configuration #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct Config { @@ -25,11 +25,11 @@ pub struct Config { /// Configuration for the GitHub comment filter pub github_comment_filter: Option, /// Configuration for the GitHub comment poster - pub github_comment_poster: Option, + pub github_comment_poster: Option, /// Configuration for the mass rebuilder pub mass_rebuilder: Option, - /// Configuration for the builder - pub builder: Option, + /// Configuration for the hydra evaluator integration + pub hydra_evaluator: Option, /// Configuration for the log message collector pub log_message_collector: Option, /// Configuration for the stats server @@ -91,7 +91,7 @@ pub struct GithubCommentFilter { /// Configuration for the GitHub comment poster #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] -pub struct GithubCommentPoster { +pub struct GitHubCommentPoster { /// RabbitMQ broker to connect to pub rabbitmq: RabbitMqConfig, } @@ -104,12 +104,16 @@ pub struct MassRebuilder { pub rabbitmq: RabbitMqConfig, } -/// Configuration for the builder -#[derive(serde::Serialize, serde::Deserialize, Debug)] +/// Configuration for the hydra evaluator integration +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] #[serde(deny_unknown_fields)] -pub struct Builder { +pub struct HydraEvaluatorConfig { /// RabbitMQ broker to connect to pub rabbitmq: RabbitMqConfig, + /// Queue-runner gRPC endpoint + pub gateway_endpoint: String, + /// Jobset ID to inject builds into + pub jobset_id: i32, } /// Configuration for the log message collector @@ -126,6 +130,8 @@ pub struct LogMessageCollector { #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct Stats { + /// Listen host/port + pub listen: String, /// RabbitMQ broker to connect to pub rabbitmq: RabbitMqConfig, } @@ -169,6 +175,17 @@ pub struct GithubAppConfig { pub oauth_client_secret_file: PathBuf, } +impl GithubAppConfig { + fn app_auth(&self) -> AppAuth { + let pem = std::fs::read_to_string(&self.private_key).expect("Unable to read private key"); + AppAuth { + app_id: self.app_id.into(), + key: jsonwebtoken::EncodingKey::from_rsa_pem(pem.as_bytes()) + .expect("Invalid private key"), + } + } +} + const fn default_instance() -> u8 { 1 } @@ -226,34 +243,24 @@ impl Config { acl::Acl::new(repos, trusted_users) } - pub fn github(&self) -> Github { - let token = std::fs::read_to_string( - self.github_app - .clone() - .expect("No GitHub app configured") - .oauth_client_secret_file, - ) - .expect("Couldn't read from GitHub app token"); - let token = token.trim(); - Github::new( - "github.com/NixOS/ofborg", - Credentials::Client( - self.github_app - .clone() - .expect("No GitHub app configured") - .oauth_client_id, - token.to_owned(), - ), - ) - .expect("Unable to create a github client instance") + pub fn github(&self) -> Octocrab { + let app_auth = self + .github_app + .as_ref() + .map(|app| app.app_auth()) + .expect("No GitHub app configured"); + Octocrab::builder() + .app(app_auth.app_id, app_auth.key) + .build() + .expect("Unable to create a github client instance") } - pub fn github_app_vendingmachine(&self) -> GithubAppVendingMachine { - GithubAppVendingMachine { - conf: self.github_app.clone().unwrap(), + pub fn github_app_vendingmachine(&self) -> Option { + Some(GithubAppVendingMachine { + conf: self.github_app.clone()?, id_cache: HashMap::new(), client_cache: HashMap::new(), - } + }) } pub fn nix(&self) -> Nix { @@ -307,8 +314,8 @@ pub fn load(filename: &Path) -> Config { pub struct GithubAppVendingMachine { conf: GithubAppConfig, - id_cache: HashMap<(String, String), Option>, - client_cache: HashMap, + id_cache: HashMap<(String, String), Option>, + client_cache: HashMap, } impl GithubAppVendingMachine { @@ -316,54 +323,56 @@ impl GithubAppVendingMachine { "github.com/NixOS/ofborg (app)" } - fn jwt(&self) -> JWTCredentials { - let pem = rustls_pki_types::PrivatePkcs1KeyDer::from_pem_file(&self.conf.private_key) - .expect("Unable to read private key"); - let private_key_der = pem.secret_pkcs1_der().to_vec(); - JWTCredentials::new(self.conf.app_id, private_key_der) - .expect("Unable to create JWTCredentials") - } + async fn install_id_for_repo(&mut self, owner: &str, repo: &str) -> Option { + let key = (owner.to_owned(), repo.to_owned()); - async fn install_id_for_repo(&mut self, owner: &str, repo: &str) -> Option { - let useragent = self.useragent(); - let jwt = self.jwt(); + if let Some(Some(id)) = self.id_cache.get(&key) { + return Some(*id); + } - let key = (owner.to_owned(), repo.to_owned()); + info!("Looking up install ID for {}/{}", owner, repo); - match self.id_cache.entry(key) { - Entry::Occupied(entry) => *entry.get(), - Entry::Vacant(entry) => { - info!("Looking up install ID for {}/{}", owner, repo); - - let lookup_gh = Github::new(useragent, Credentials::JWT(jwt)).unwrap(); - - let v = match lookup_gh.app().find_repo_installation(owner, repo).await { - Ok(install_id) => { - debug!("Received install ID {:?}", install_id); - Some(install_id.id) - } - Err(e) => { - warn!("Error during install ID lookup: {:?}", e); - None - } - }; - *entry.insert(v) + let app_auth = self.conf.app_auth(); + let octocrab = Octocrab::builder() + .add_header(http::header::USER_AGENT, self.useragent().parse().unwrap()) + .app(app_auth.app_id, app_auth.key) + .build() + .expect("Unable to create app client"); + + match octocrab + .apps() + .get_repository_installation(owner, repo) + .await + { + Ok(installation) => { + debug!("Received install ID {:?}", installation.id); + let id = installation.id; + self.id_cache.insert(key, Some(id)); + Some(id) + } + Err(e) => { + warn!("Error during install ID lookup: {:?}", e); + None } } } - pub async fn for_repo<'a>(&'a mut self, owner: &str, repo: &str) -> Option<&'a Github> { - let useragent = self.useragent(); - let jwt = self.jwt(); + pub async fn for_repo<'a>(&'a mut self, owner: &str, repo: &str) -> Option<&'a Octocrab> { let install_id = self.install_id_for_repo(owner, repo).await?; - Some(self.client_cache.entry(install_id).or_insert_with(|| { - Github::new( - useragent, - Credentials::InstallationToken(InstallationTokenGenerator::new(install_id, jwt)), - ) - .expect("Unable to create a github client instance") - })) + if !self.client_cache.contains_key(&install_id) { + let app_auth = self.conf.app_auth(); + let client = Octocrab::builder() + .add_header(http::header::USER_AGENT, self.useragent().parse().unwrap()) + .app(app_auth.app_id, app_auth.key) + .build() + .expect("Unable to create app client") + .installation(install_id) + .expect("Unable to create installation client"); + self.client_cache.insert(install_id, client); + } + + self.client_cache.get(&install_id) } } diff --git a/ofborg/src/easylapin.rs b/ofborg/src/easylapin.rs index 08b7f7a1..549b5abd 100644 --- a/ofborg/src/easylapin.rs +++ b/ofborg/src/easylapin.rs @@ -1,15 +1,6 @@ use std::pin::Pin; use std::sync::Arc; -use crate::config::RabbitMqConfig; -use crate::easyamqp::{ - BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, - QueueConfig, -}; -use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker}; -use crate::ofborg; -use crate::worker::{Action, SimpleWorker}; - use lapin::message::Delivery; use lapin::options::{ BasicAckOptions, BasicConsumeOptions, BasicNackOptions, BasicPublishOptions, BasicQosOptions, @@ -17,9 +8,18 @@ use lapin::options::{ }; use lapin::types::FieldTable; use lapin::{BasicProperties, Channel, Connection, ConnectionProperties, ExchangeKind}; -use tokio_stream::StreamExt; +use tokio_stream::StreamExt as _; use tracing::{debug, trace}; +use crate::config::RabbitMqConfig; +use crate::easyamqp::{ + BindQueueConfig, ChannelExt, ConsumeConfig, ConsumerExt, ExchangeConfig, ExchangeType, + QueueConfig, +}; +use crate::notifyworker::{NotificationReceiver, SimpleNotifyWorker}; +use crate::ofborg; +use crate::worker::{Action, SimpleWorker}; + pub async fn from_config(cfg: &RabbitMqConfig) -> Result { let opts = ConnectionProperties::default() .with_client_property("ofborg_version".into(), ofborg::VERSION.into()); diff --git a/ofborg/src/evalchecker.rs b/ofborg/src/evalchecker.rs index d7af769b..7a56ae64 100644 --- a/ofborg/src/evalchecker.rs +++ b/ofborg/src/evalchecker.rs @@ -1,8 +1,8 @@ -use crate::nix; - use std::fs::File; use std::path::Path; +use crate::nix; + pub struct EvalChecker { name: String, op: nix::Operation, diff --git a/ofborg/src/files.rs b/ofborg/src/files.rs index 9e329d83..ee5a4207 100644 --- a/ofborg/src/files.rs +++ b/ofborg/src/files.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::Read; +use std::io::Read as _; pub fn file_to_str(f: &mut File) -> String { let mut buffer = Vec::new(); diff --git a/ofborg/src/ghevent/mod.rs b/ofborg/src/ghevent/mod.rs index 24375880..98183242 100644 --- a/ofborg/src/ghevent/mod.rs +++ b/ofborg/src/ghevent/mod.rs @@ -5,5 +5,5 @@ mod pullrequestevent; pub use self::common::{Comment, GenericWebhook, Issue, Repository, User}; pub use self::issuecomment::{IssueComment, IssueCommentAction}; pub use self::pullrequestevent::{ - PullRequest, PullRequestAction, PullRequestEvent, PullRequestState, + PullRequest, PullRequestAction, PullRequestEvent, PullRequestRef, PullRequestState, }; diff --git a/ofborg/src/github.rs b/ofborg/src/github.rs new file mode 100644 index 00000000..095a4cbb --- /dev/null +++ b/ofborg/src/github.rs @@ -0,0 +1,100 @@ +use octocrab::Octocrab; +use tracing::info; + +use crate::commitstatus::CommitStatusError; + +#[derive(Clone)] +pub struct GithubRepo { + octocrab: Octocrab, + owner: String, + repo: String, +} + +impl GithubRepo { + pub fn new(octocrab: Octocrab, owner: impl Into, repo: impl Into) -> Self { + Self { + octocrab, + owner: owner.into(), + repo: repo.into(), + } + } + + pub fn owner(&self) -> &str { + &self.owner + } + + pub fn repo(&self) -> &str { + &self.repo + } + + pub fn repos(&self) -> octocrab::repos::RepoHandler<'_> { + self.octocrab.repos(&self.owner, &self.repo) + } + + pub fn issues(&self) -> octocrab::issues::IssueHandler<'_> { + self.octocrab.issues(&self.owner, &self.repo) + } + + pub fn checks(&self) -> octocrab::checks::ChecksHandler<'_> { + self.octocrab.checks(&self.owner, &self.repo) + } + + pub fn pulls(&self) -> octocrab::pulls::PullRequestHandler<'_> { + self.octocrab.pulls(&self.owner, &self.repo) + } + + pub async fn update_labels( + &self, + issue_number: u64, + add: &[String], + remove: &[String], + ) -> Result<(), CommitStatusError> { + let issue = self.issues().get(issue_number).await?; + + let existing: Vec = issue.labels.iter().map(|l| l.name.clone()).collect(); + + let to_add: Vec = add + .iter() + .filter(|l| !existing.contains(l)) + .cloned() + .collect(); + + let to_remove: Vec = remove + .iter() + .filter(|l| existing.contains(l)) + .cloned() + .collect(); + + info!("Labeling issue #{issue_number}: + {to_add:?} , - {to_remove:?}, = {existing:?}"); + + if !to_add.is_empty() { + self.issues().add_labels(issue_number, &to_add).await?; + } + + for label in to_remove { + self.issues().remove_label(issue_number, &label).await?; + } + + Ok(()) + } + + pub async fn get_prefix(&self, sha: &str) -> Result<&'static str, CommitStatusError> { + let mut page: octocrab::Page = + self.repos().list_statuses(sha.to_string()).send().await?; + + loop { + if page.items.iter().any(|s| { + s.context + .as_ref() + .is_some_and(|c| c.starts_with("grahamcofborg-")) + }) { + return Ok("grahamcofborg"); + } + + match self.octocrab.get_page(&page.next).await? { + Some(next_page) => page = next_page, + None => return Ok("ofborg"), + } + } + } +} diff --git a/ofborg/src/lib.rs b/ofborg/src/lib.rs index eb5da459..1383efc7 100644 --- a/ofborg/src/lib.rs +++ b/ofborg/src/lib.rs @@ -7,8 +7,7 @@ use std::env; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::prelude::*; +use tracing_subscriber::{EnvFilter, prelude::*}; pub mod acl; pub mod asynccmd; @@ -22,6 +21,7 @@ pub mod easylapin; pub mod evalchecker; pub mod files; pub mod ghevent; +pub mod github; pub mod locks; pub mod maintainers; pub mod message; @@ -34,7 +34,15 @@ pub mod systems; pub mod tagger; pub mod tasks; pub mod test_scratch; +pub mod test_utils; pub mod worker; + +use async_trait::async_trait; + +#[async_trait] +pub trait MessagePublisher: Send + Sync { + async fn publish(&self, exchange: &str, routing_key: &str, body: &[u8]) -> anyhow::Result<()>; +} pub mod writetoline; pub mod ofborg { diff --git a/ofborg/src/locks.rs b/ofborg/src/locks.rs index d1d2ee47..7d99b069 100644 --- a/ofborg/src/locks.rs +++ b/ofborg/src/locks.rs @@ -1,9 +1,9 @@ -use fs2::FileExt; - use std::fs; use std::io::Error; use std::path::PathBuf; +use fs2::FileExt; + pub trait Lockable { fn lock_path(&self) -> PathBuf; diff --git a/ofborg/src/maintainers.rs b/ofborg/src/maintainers.rs index ff1bec0c..33cfe871 100644 --- a/ofborg/src/maintainers.rs +++ b/ofborg/src/maintainers.rs @@ -1,11 +1,11 @@ -use crate::nix::Nix; - -use tempfile::NamedTempFile; - use std::collections::{HashMap, HashSet}; use std::io::Write; use std::path::Path; +use tempfile::NamedTempFile; + +use crate::nix::Nix; + #[derive(serde::Deserialize, Debug, Eq, PartialEq)] pub struct ImpactedMaintainers(HashMap>); pub struct MaintainersByPackage(pub HashMap>); diff --git a/ofborg/src/message/buildjob.rs b/ofborg/src/message/buildjob.rs index b09eae58..18c970e1 100644 --- a/ofborg/src/message/buildjob.rs +++ b/ofborg/src/message/buildjob.rs @@ -1,7 +1,7 @@ use crate::commentparser::Subset; use crate::message::{Pr, Repo}; -#[derive(serde::Serialize, serde::Deserialize, Debug)] +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub struct BuildJob { pub repo: Repo, pub pr: Pr, diff --git a/ofborg/src/message/buildresult.rs b/ofborg/src/message/buildresult.rs index 70d8d1ea..017780fb 100644 --- a/ofborg/src/message/buildresult.rs +++ b/ofborg/src/message/buildresult.rs @@ -1,6 +1,6 @@ -use crate::message::{Pr, Repo}; +use octocrab::models::workflows::Conclusion; -use hubcaps::checks::Conclusion; +use crate::message::{Pr, Repo}; #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq, Eq)] pub enum BuildStatus { diff --git a/ofborg/src/message/hydra_eval_job.rs b/ofborg/src/message/hydra_eval_job.rs new file mode 100644 index 00000000..ed12ea88 --- /dev/null +++ b/ofborg/src/message/hydra_eval_job.rs @@ -0,0 +1,14 @@ +use crate::message::{Pr, Repo}; + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct HydraEvalJob { + pub repo: Repo, + pub pr: Pr, + pub drv_paths: Vec, + pub request_id: String, + pub jobset_id: i32, +} + +pub fn from(data: &[u8]) -> Result { + serde_json::from_slice(data) +} diff --git a/ofborg/src/message/mod.rs b/ofborg/src/message/mod.rs index 03551cd1..41efc579 100644 --- a/ofborg/src/message/mod.rs +++ b/ofborg/src/message/mod.rs @@ -3,5 +3,6 @@ pub mod buildlogmsg; pub mod buildresult; mod common; pub mod evaluationjob; +pub mod hydra_eval_job; pub use self::common::{Pr, Repo}; diff --git a/ofborg/src/nix.rs b/ofborg/src/nix.rs index 77aece6d..e71723ca 100644 --- a/ofborg/src/nix.rs +++ b/ofborg/src/nix.rs @@ -1,18 +1,16 @@ -use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd}; -use crate::message::buildresult::BuildStatus; -use crate::ofborg::partition_result; - use std::collections::HashMap; -use std::env; use std::ffi::OsStr; -use std::fmt; -use std::fs; -use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::io::{BufRead as _, BufReader, Seek as _, SeekFrom}; use std::path::Path; use std::process::{Command, Stdio}; +use std::{env, fmt, fs}; use tempfile::tempfile; +use crate::asynccmd::{AsyncCmd, SpawnedAsyncCmd}; +use crate::message::buildresult::BuildStatus; +use crate::ofborg::partition_result; + #[allow(clippy::upper_case_acronyms)] #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum File { @@ -60,8 +58,8 @@ impl Operation { "--no-out-link", "--keep-going", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::QueryPackagesJson => { @@ -70,8 +68,8 @@ impl Operation { "--available", "--json", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::QueryPackagesOutputs => { @@ -82,8 +80,8 @@ impl Operation { "--attr-path", "--out-path", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::NoOp { ref operation } => { @@ -95,12 +93,12 @@ impl Operation { "--strict", "--json", "--option", - "extra-experimental-features", - "no-url-literals", + "lint-url-literals", + "fatal", ]); } Operation::Instantiate => { - command.args(["--option", "extra-experimental-features", "no-url-literals"]); + command.args(["--option", "lint-url-literals", "fatal"]); } _ => (), }; diff --git a/ofborg/src/tasks/build.rs b/ofborg/src/tasks/build.rs index 9ea59de3..e7c01ded 100644 --- a/ofborg/src/tasks/build.rs +++ b/ofborg/src/tasks/build.rs @@ -1,3 +1,10 @@ +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use tracing::{Instrument, debug, debug_span, error, info}; +use uuid::Uuid; + use crate::checkout; use crate::commentparser; use crate::message::buildresult::{BuildResult, BuildStatus, V1Tag}; @@ -6,13 +13,6 @@ use crate::nix; use crate::notifyworker; use crate::worker; -use std::collections::VecDeque; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; - -use tracing::{debug, debug_span, error, info}; -use uuid::Uuid; - pub struct BuildWorker { cloner: checkout::CachedCloner, nix: nix::Nix, @@ -303,114 +303,116 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { >, ) { let span = debug_span!("job", pr = ?job.pr.number); - let _enter = span.enter(); - - let actions = self.actions(job, notifier); - - if actions.job.attrs.is_empty() { - debug!("No attrs to build"); - actions.nothing_to_do().await; - return; - } + async { + let actions = self.actions(job, notifier); - info!( - "Working on https://github.com/{}/pull/{}", - actions.job.repo.full_name, actions.job.pr.number - ); - let project = self.cloner.project( - &actions.job.repo.full_name, - actions.job.repo.clone_url.clone(), - ); - let co = project - .clone_for("builder".to_string(), self.identity.clone()) - .unwrap(); - - let target_branch = match actions.job.pr.target_branch.clone() { - Some(x) => x, - None => String::from("origin/master"), - }; - - let buildfile = match actions.job.subset { - Some(commentparser::Subset::NixOS) => nix::File::ReleaseNixOS, - _ => nix::File::DefaultNixpkgs, - }; + if actions.job.attrs.is_empty() { + debug!("No attrs to build"); + actions.nothing_to_do().await; + return; + } - let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap(); + info!( + "Working on https://github.com/{}/pull/{}", + actions.job.repo.full_name, actions.job.pr.number + ); + let project = self.cloner.project( + &actions.job.repo.full_name, + actions.job.repo.clone_url.clone(), + ); + let co = project + .clone_for("builder".to_string(), self.identity.clone()) + .unwrap(); + + let target_branch = match actions.job.pr.target_branch.clone() { + Some(x) => x, + None => String::from("origin/master"), + }; + + let buildfile = match actions.job.subset { + Some(commentparser::Subset::NixOS) => nix::File::ReleaseNixOS, + _ => nix::File::DefaultNixpkgs, + }; + + let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap(); + + if co.fetch_pr(actions.job.pr.number).is_err() { + info!("Failed to fetch {}", actions.job.pr.number); + actions.pr_head_missing().await; + return; + } - if co.fetch_pr(actions.job.pr.number).is_err() { - info!("Failed to fetch {}", actions.job.pr.number); - actions.pr_head_missing().await; - return; - } + if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { + info!("Commit {} doesn't exist", actions.job.pr.head_sha); + actions.commit_missing().await; + return; + } - if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { - info!("Commit {} doesn't exist", actions.job.pr.head_sha); - actions.commit_missing().await; - return; - } + if co.merge_commit(actions.job.pr.head_sha.as_ref()).is_err() { + info!("Failed to merge {}", actions.job.pr.head_sha); + actions.merge_failed().await; + return; + } - if co.merge_commit(actions.job.pr.head_sha.as_ref()).is_err() { - info!("Failed to merge {}", actions.job.pr.head_sha); - actions.merge_failed().await; - return; - } + info!( + "Got path: {:?}, determining which ones we can build ", + refpath + ); + let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs( + refpath.as_ref(), + buildfile, + actions.job.attrs.clone(), + ); + + let cannot_build_attrs: Vec = cannot_build + .clone() + .into_iter() + .map(|(attr, _)| attr) + .collect(); + + info!( + "Can build: '{}', Cannot build: '{}'", + can_build.join(", "), + cannot_build_attrs.join(", ") + ); + + actions + .log_started(can_build.clone(), cannot_build_attrs.clone()) + .await; + actions.log_instantiation_errors(cannot_build).await; - info!( - "Got path: {:?}, determining which ones we can build ", - refpath - ); - let (can_build, cannot_build) = self.nix.safely_partition_instantiable_attrs( - refpath.as_ref(), - buildfile, - actions.job.attrs.clone(), - ); + if can_build.is_empty() { + actions.build_not_attempted(cannot_build_attrs).await; + return; + } - let cannot_build_attrs: Vec = cannot_build - .clone() - .into_iter() - .map(|(attr, _)| attr) - .collect(); - - info!( - "Can build: '{}', Cannot build: '{}'", - can_build.join(", "), - cannot_build_attrs.join(", ") - ); + let mut spawned = + self.nix + .safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone()); - actions - .log_started(can_build.clone(), cannot_build_attrs.clone()) - .await; - actions.log_instantiation_errors(cannot_build).await; + while let Ok(line) = spawned.get_next_line() { + actions.log_line(line).await; + } - if can_build.is_empty() { - actions.build_not_attempted(cannot_build_attrs).await; - return; - } + let status = nix::wait_for_build_status(spawned); - let mut spawned = - self.nix - .safely_build_attrs_async(refpath.as_ref(), buildfile, can_build.clone()); + info!("ok built ({:?}), building", status); + info!("Lines:"); + info!("-----8<-----"); + actions + .log_snippet() + .iter() + .inspect(|x| info!("{}", x)) + .next_back(); + info!("----->8-----"); - while let Ok(line) = spawned.get_next_line() { - actions.log_line(line).await; + actions + .build_finished(status, can_build, cannot_build_attrs) + .await; + info!("Build done!"); } - - let status = nix::wait_for_build_status(spawned); - - info!("ok built ({:?}), building", status); - info!("Lines:"); - info!("-----8<-----"); - actions - .log_snippet() - .iter() - .inspect(|x| info!("{}", x)) - .next_back(); - info!("----->8-----"); - - actions - .build_finished(status, can_build, cannot_build_attrs) - .await; - info!("Build done!"); + .instrument(span) + .await } } diff --git a/ofborg/src/tasks/eval/mod.rs b/ofborg/src/tasks/eval/mod.rs index b312db9a..f7baa954 100644 --- a/ofborg/src/tasks/eval/mod.rs +++ b/ofborg/src/tasks/eval/mod.rs @@ -4,12 +4,14 @@ pub use self::nixpkgs::NixpkgsStrategy; use crate::checkout::CachedProjectCo; use crate::commitstatus::{CommitStatus, CommitStatusError}; use crate::evalchecker::EvalChecker; +use crate::github::GithubRepo; use crate::message::buildjob::BuildJob; use std::path::Path; pub trait EvaluationStrategy { - fn pre_clone(&mut self) -> impl std::future::Future>; + fn pre_clone(&mut self, repo: &GithubRepo) + -> impl std::future::Future>; fn on_target_branch( &mut self, diff --git a/ofborg/src/tasks/eval/nixpkgs.rs b/ofborg/src/tasks/eval/nixpkgs.rs index 8f0fd7d7..9b8f8858 100644 --- a/ofborg/src/tasks/eval/nixpkgs.rs +++ b/ofborg/src/tasks/eval/nixpkgs.rs @@ -1,17 +1,18 @@ +use std::path::Path; + +use octocrab::models::StatusState; +use regex::Regex; +use tracing::warn; +use uuid::Uuid; + use crate::checkout::CachedProjectCo; use crate::commentparser::Subset; use crate::commitstatus::CommitStatus; use crate::evalchecker::EvalChecker; +use crate::github::GithubRepo; use crate::message::buildjob::BuildJob; use crate::message::evaluationjob::EvaluationJob; use crate::tasks::eval::{EvaluationComplete, EvaluationStrategy, StepResult}; -use crate::tasks::evaluate::update_labels; - -use std::path::Path; - -use hubcaps::issues::IssueRef; -use regex::Regex; -use uuid::Uuid; const TITLE_LABELS: [(&str, &str); 4] = [ ("bsd", "6.topic: bsd"), @@ -35,23 +36,26 @@ fn label_from_title(title: &str) -> Vec { pub struct NixpkgsStrategy<'a> { job: &'a EvaluationJob, - issue_ref: &'a IssueRef, + issue: Option<&'a octocrab::models::issues::Issue>, touched_packages: Option>, } impl<'a> NixpkgsStrategy<'a> { - pub fn new(job: &'a EvaluationJob, issue_ref: &'a IssueRef) -> NixpkgsStrategy<'a> { + pub fn new( + job: &'a EvaluationJob, + issue: Option<&'a octocrab::models::issues::Issue>, + ) -> NixpkgsStrategy<'a> { Self { job, - issue_ref, + issue, touched_packages: None, } } - async fn tag_from_title(&self) { - let title = match self.issue_ref.get().await { - Ok(issue) => issue.title.to_lowercase(), - Err(_) => return, + async fn tag_from_title(&self, repo: &GithubRepo) { + let title = match self.issue { + Some(issue) => issue.title.to_lowercase(), + None => return, }; let labels = label_from_title(&title); @@ -60,7 +64,10 @@ impl<'a> NixpkgsStrategy<'a> { return; } - update_labels(self.issue_ref, &labels, &[]).await; + let issue_number = self.issue.map(|i| i.number).unwrap_or(self.job.pr.number); + if let Err(e) = repo.update_labels(issue_number, &labels, &[]).await { + warn!("Failed to update labels on #{issue_number}: {e:?}"); + } } fn check_outpaths_before(&mut self, _dir: &Path) -> StepResult<()> { @@ -104,17 +111,14 @@ impl<'a> NixpkgsStrategy<'a> { } impl EvaluationStrategy for NixpkgsStrategy<'_> { - async fn pre_clone(&mut self) -> StepResult<()> { - self.tag_from_title().await; + async fn pre_clone(&mut self, repo: &GithubRepo) -> StepResult<()> { + self.tag_from_title(repo).await; Ok(()) } async fn on_target_branch(&mut self, dir: &Path, status: &mut CommitStatus) -> StepResult<()> { status - .set_with_description( - "Checking original out paths", - hubcaps::statuses::State::Pending, - ) + .set_with_description("Checking original out paths", StatusState::Pending) .await?; self.check_outpaths_before(dir)?; @@ -132,7 +136,7 @@ impl EvaluationStrategy for NixpkgsStrategy<'_> { async fn after_merge(&mut self, status: &mut CommitStatus) -> StepResult<()> { status - .set_with_description("Checking new out paths", hubcaps::statuses::State::Pending) + .set_with_description("Checking new out paths", StatusState::Pending) .await?; self.check_outpaths_after()?; @@ -148,10 +152,7 @@ impl EvaluationStrategy for NixpkgsStrategy<'_> { status: &mut CommitStatus, ) -> StepResult { status - .set_with_description( - "Calculating Changed Outputs", - hubcaps::statuses::State::Pending, - ) + .set_with_description("Calculating Changed Outputs", StatusState::Pending) .await?; let builds = self.queue_builds()?; diff --git a/ofborg/src/tasks/evaluate.rs b/ofborg/src/tasks/evaluate.rs index f34cf2b5..d4caf39a 100644 --- a/ofborg/src/tasks/evaluate.rs +++ b/ofborg/src/tasks/evaluate.rs @@ -1,44 +1,52 @@ /// This is what evaluates every pull-request -use crate::acl::Acl; -use crate::checkout; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::time::Instant; + +use futures::stream::StreamExt; +use octocrab::{Octocrab, models::StatusState}; +use tracing::{Instrument, debug_span, error, info, warn}; + use crate::commitstatus::{CommitStatus, CommitStatusError}; use crate::config::GithubAppVendingMachine; -use crate::message::{buildjob, evaluationjob}; +use crate::github::GithubRepo; +use crate::message::{buildjob, evaluationjob, hydra_eval_job}; +use crate::nix; use crate::stats::{self, Event}; -use crate::systems; use crate::tasks::eval; use crate::tasks::eval::EvaluationStrategy; -use crate::worker; -use futures::stream::StreamExt; -use futures_util::TryFutureExt; - -use std::path::Path; -use std::time::Instant; - -use tracing::{debug_span, error, info, warn}; +use crate::{checkout, worker}; +use uuid::Uuid; pub struct EvaluationWorker { cloner: checkout::CachedCloner, - github_vend: tokio::sync::RwLock, - acl: Acl, + github_vend: Option>, identity: String, events: E, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, } impl EvaluationWorker { + #[allow(clippy::too_many_arguments)] pub fn new( cloner: checkout::CachedCloner, - github_vend: GithubAppVendingMachine, - acl: Acl, + github_vend: Option, identity: String, events: E, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, ) -> EvaluationWorker { EvaluationWorker { cloner, - github_vend: tokio::sync::RwLock::new(github_vend), - acl, + github_vend: github_vend.map(tokio::sync::RwLock::new), identity, events, + hydra_eval_queue, + hydra_eval_nix, + hydra_eval_jobset_id, } } } @@ -71,57 +79,93 @@ impl worker::SimpleWorker for EvaluationWorker async fn consumer(&mut self, job: &evaluationjob::EvaluationJob) -> worker::Actions { let span = debug_span!("job", pr = ?job.pr.number); - let _enter = span.enter(); - - let mut vending_machine = self.github_vend.write().await; - - let github_client = vending_machine - .for_repo(&job.repo.owner, &job.repo.name) + async { + let github_client = if let Some(github_vend) = self.github_vend.as_ref() { + let mut vending_machine = github_vend.write().await; + match vending_machine + .for_repo(&job.repo.owner, &job.repo.name) + .await + { + Some(client) => Some(client.clone()), + None => { + error!( + "Failed to get a github client token for {}/{}", + job.repo.owner, job.repo.name + ); + return vec![worker::Action::NackRequeue]; + } + } + } else { + None + }; + + OneEval::new( + github_client, + &mut self.events, + &self.identity, + &self.cloner, + job, + self.hydra_eval_queue.clone(), + self.hydra_eval_nix.clone(), + self.hydra_eval_jobset_id, + ) + .worker_actions() .await - .expect("Failed to get a github client token"); - - OneEval::new( - github_client, - &self.acl, - &mut self.events, - &self.identity, - &self.cloner, - job, - ) - .worker_actions() + } + .instrument(span) .await } } struct OneEval<'a, E> { - client_app: &'a hubcaps::Github, - repo: hubcaps::repositories::Repository, - acl: &'a Acl, + repo: GithubRepo, + enable_publish: bool, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, + prefix: Option<&'static str>, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, } impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { #[allow(clippy::too_many_arguments)] + #[allow(clippy::borrow_as_ptr)] fn new( - client_app: &'a hubcaps::Github, - acl: &'a Acl, + octocrab: Option, events: &'a mut E, identity: &'a str, cloner: &'a checkout::CachedCloner, job: &'a evaluationjob::EvaluationJob, + hydra_eval_queue: Option, + hydra_eval_nix: Option, + hydra_eval_jobset_id: Option, ) -> OneEval<'a, E> { - let repo = client_app.repo(job.repo.owner.clone(), job.repo.name.clone()); + let (repo, enable_publish) = if let Some(octocrab) = octocrab { + ( + GithubRepo::new(octocrab, job.repo.owner.clone(), job.repo.name.clone()), + true, + ) + } else { + let octocrab = Octocrab::builder().build().unwrap(); + ( + GithubRepo::new(octocrab, job.repo.owner.clone(), job.repo.name.clone()), + false, + ) + }; OneEval { - client_app, repo, - acl, + enable_publish, events, identity, cloner, job, + prefix: None, + hydra_eval_queue, + hydra_eval_nix, + hydra_eval_jobset_id, } } @@ -133,52 +177,53 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { &self, description: String, url: Option, - state: hubcaps::statuses::State, + state: StatusState, ) -> Result<(), CommitStatusError> { - let description = if description.len() >= 140 { - warn!( - "description is over 140 char; truncating: {:?}", - &description - ); - description.chars().take(140).collect() - } else { - description - }; - let repo = self - .client_app - .repo(self.job.repo.owner.clone(), self.job.repo.name.clone()); - let prefix = get_prefix(repo.statuses(), &self.job.pr.head_sha).await?; - - let mut builder = hubcaps::statuses::StatusOptions::builder(state); - builder.context(format!("{prefix}-eval")); - builder.description(description.clone()); - - if let Some(url) = url { - builder.target_url(url); + if !self.enable_publish { + return Ok(()); } + let prefix = self + .prefix + .expect("prefix should have been set in worker_actions"); + info!( "Updating status on {}:{} -> {}", &self.job.pr.number, &self.job.pr.head_sha, &description ); - self.repo - .statuses() - .create(&self.job.pr.head_sha, &builder.build()) - .map_ok(|_| ()) - .map_err(|e| CommitStatusError::from(e)) - .await + let status = CommitStatus::new( + self.repo.clone(), + self.job.pr.head_sha.clone(), + format!("{prefix}-eval"), + description, + url, + ); + + status.set(state).await } async fn worker_actions(&mut self) -> worker::Actions { + self.prefix = Some(if self.enable_publish { + match self.repo.get_prefix(&self.job.pr.head_sha).await { + Ok(p) => p, + Err(e) => { + error!("Failed to determine commit status prefix: {:?}", e); + return self.actions().retry_later(self.job); + } + } + } else { + "ofborg" + }); + let eval_result = match self.evaluate_job().await { Ok(v) => Ok(v), Err(eval_error) => match eval_error { // Handle error cases which expect us to post statuses // to github. Convert Eval Errors in to Result<_, CommitStatusWrite> - EvalWorkerError::EvalError(eval::Error::Fail(msg)) => Err(self - .update_status(msg, None, hubcaps::statuses::State::Failure) - .await), + EvalWorkerError::EvalError(eval::Error::Fail(msg)) => { + Err(self.update_status(msg, None, StatusState::Failure).await) + } EvalWorkerError::EvalError(eval::Error::CommitStatusWrite(e)) => Err(Err(e)), EvalWorkerError::CommitStatusWrite(e) => Err(Err(e)), }, @@ -186,8 +231,15 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { match eval_result { Ok(eval_actions) => { - let issue_ref = self.repo.issue(self.job.pr.number); - update_labels(&issue_ref, &[], &[String::from("ofborg-internal-error")]).await; + if self.enable_publish + && let Ok(issue) = self.repo.issues().get(self.job.pr.number).await + && let Err(e) = self + .repo + .update_labels(issue.number, &[], &[String::from("ofborg-internal-error")]) + .await + { + warn!("Failed to update labels: {e:?}"); + } eval_actions } @@ -195,98 +247,84 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { // There was an error during eval, but we successfully // updated the PR. - let issue_ref = self.repo.issue(self.job.pr.number); - update_labels(&issue_ref, &[], &[String::from("ofborg-internal-error")]).await; + if self.enable_publish + && let Ok(issue) = self.repo.issues().get(self.job.pr.number).await + && let Err(e) = self + .repo + .update_labels(issue.number, &[], &[String::from("ofborg-internal-error")]) + .await + { + warn!("Failed to update labels: {e:?}"); + } self.actions().skip(self.job) } - Err(Err(CommitStatusError::ExpiredCreds(e))) => { - error!("Failed writing commit status: creds expired: {:?}", e); + Err(Err(CommitStatusError::OctocrabError(e))) => { + error!("Failed writing commit status: {:?}", e); self.actions().retry_later(self.job) } Err(Err(CommitStatusError::InternalError(e))) => { error!("Failed writing commit status: internal error: {:?}", e); self.actions().retry_later(self.job) } - Err(Err(CommitStatusError::MissingSha(e))) => { - error!( - "Failed writing commit status: commit sha was force-pushed away: {:?}", - e - ); - self.actions().skip(self.job) - } - - Err(Err(CommitStatusError::Error(cswerr))) => { - error!( - "Internal error writing commit status: {:?}, marking internal error", - cswerr - ); - let issue_ref = self.repo.issue(self.job.pr.number); - update_labels(&issue_ref, &[String::from("ofborg-internal-error")], &[]).await; - - self.actions().skip(self.job) - } } } async fn evaluate_job(&mut self) -> Result { let job = self.job; - let repo = self - .client_app - .repo(self.job.repo.owner.clone(), self.job.repo.name.clone()); - let issue_ref = repo.issue(job.pr.number); - let auto_schedule_build_archs: Vec; - - match issue_ref.get().await { - Ok(iss) => { - if iss.state == "closed" { - self.events.notify(Event::IssueAlreadyClosed).await; - info!("Skipping {} because it is closed", job.pr.number); - return Ok(self.actions().skip(job)); + let issue = if self.enable_publish { + let issue_result = self.repo.issues().get(job.pr.number).await; + + match &issue_result { + Ok(iss) => { + if iss.state == octocrab::models::IssueState::Closed { + self.events.notify(Event::IssueAlreadyClosed).await; + info!("Skipping {} because it is closed", job.pr.number); + return Ok(self.actions().skip(job)); + } } - if issue_is_wip(&iss) { - auto_schedule_build_archs = vec![]; - } else { - auto_schedule_build_archs = self.acl.build_job_architectures_for_user_repo( - &iss.user.login, - &job.repo.full_name, - ); + Err(e) => { + self.events.notify(Event::IssueFetchFailed).await; + error!("Error fetching {}!", job.pr.number); + error!("E: {:?}", e); + return Ok(self.actions().skip(job)); } - } + }; - Err(e) => { - self.events.notify(Event::IssueFetchFailed).await; - error!("Error fetching {}!", job.pr.number); - error!("E: {:?}", e); - return Ok(self.actions().skip(job)); - } + issue_result.ok() + } else { + None }; + let mut evaluation_strategy = eval::NixpkgsStrategy::new(job, issue.as_ref()); - let mut evaluation_strategy = eval::NixpkgsStrategy::new(job, &issue_ref); - - let prefix = get_prefix(repo.statuses(), &job.pr.head_sha).await?; + let prefix = self + .prefix + .expect("prefix should have been set in worker_actions"); let mut overall_status = CommitStatus::new( - repo.statuses(), + self.repo.clone(), job.pr.head_sha.clone(), format!("{prefix}-eval"), "Starting".to_owned(), None, ); + overall_status.set_enable_publish(self.enable_publish); overall_status - .set_with_description("Starting", hubcaps::statuses::State::Pending) + .set_with_description("Starting", StatusState::Pending) .await?; - evaluation_strategy.pre_clone().await?; + if self.enable_publish { + evaluation_strategy.pre_clone(&self.repo).await?; + } let project = self .cloner .project(&job.repo.full_name, job.repo.clone_url.clone()); overall_status - .set_with_description("Cloning project", hubcaps::statuses::State::Pending) + .set_with_description("Cloning project", StatusState::Pending) .await?; info!("Working on {}", job.pr.number); @@ -307,8 +345,8 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { overall_status .set_with_description( "The branch you have targeted is a read-only mirror for channels. \ - Please target release-* or master.", - hubcaps::statuses::State::Error, + Please target release-* or master.", + StatusState::Error, ) .await?; @@ -319,7 +357,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { overall_status .set_with_description( format!("Checking out {}", &target_branch).as_ref(), - hubcaps::statuses::State::Pending, + StatusState::Pending, ) .await?; info!("Checking out target branch {}", &target_branch); @@ -331,12 +369,12 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { ))) })?; + let target_branch_rebuild_sniff_start = Instant::now(); + evaluation_strategy .on_target_branch(Path::new(&refpath), &mut overall_status) .await?; - let target_branch_rebuild_sniff_start = Instant::now(); - self.events .notify(Event::EvaluationDuration( target_branch.clone(), @@ -348,7 +386,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { .await; overall_status - .set_with_description("Fetching PR", hubcaps::statuses::State::Pending) + .set_with_description("Fetching PR", StatusState::Pending) .await?; co.fetch_pr(job.pr.number).map_err(|e| { @@ -359,7 +397,7 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { if !co.commit_exists(job.pr.head_sha.as_ref()) { overall_status - .set_with_description("Commit not found", hubcaps::statuses::State::Error) + .set_with_description("Commit not found", StatusState::Error) .await?; info!("Commit {} doesn't exist", job.pr.head_sha); @@ -369,12 +407,12 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { evaluation_strategy.after_fetch(&co)?; overall_status - .set_with_description("Merging PR", hubcaps::statuses::State::Pending) + .set_with_description("Merging PR", StatusState::Pending) .await?; if co.merge_commit(job.pr.head_sha.as_ref()).is_err() { overall_status - .set_with_description("Failed to merge", hubcaps::statuses::State::Failure) + .set_with_description("Failed to merge", StatusState::Failure) .await?; info!("Failed to merge {}", job.pr.head_sha); @@ -386,41 +424,40 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { info!("Got path: {:?}, building", refpath); overall_status - .set_with_description("Beginning Evaluations", hubcaps::statuses::State::Pending) + .set_with_description("Beginning Evaluations", StatusState::Pending) .await?; + let enable_publish = self.enable_publish; let eval_results: bool = futures::stream::iter(evaluation_strategy.evaluation_checks()) .map(|check| { - // We need to clone or move variables into the async block - let repo_statuses = repo.statuses(); + let repo = self.repo.clone(); let head_sha = job.pr.head_sha.clone(); let refpath = refpath.clone(); async move { - let status = CommitStatus::new( - repo_statuses, + let mut status = CommitStatus::new( + repo, head_sha, format!("{prefix}-eval-{}", check.name()), check.cli_cmd(), None, ); + status.set_enable_publish(enable_publish); - status - .set(hubcaps::statuses::State::Pending) - .await - .expect("Failed to set status on eval strategy"); + if let Err(e) = status.set(StatusState::Pending).await { + warn!("Failed to set pending status on eval strategy: {e:?}"); + } let state = match check.execute(Path::new(&refpath)) { - Ok(_) => hubcaps::statuses::State::Success, - Err(_) => hubcaps::statuses::State::Failure, + Ok(_) => StatusState::Success, + Err(_) => StatusState::Failure, }; - status - .set(state.clone()) - .await - .expect("Failed to set status on eval strategy"); + if let Err(e) = status.set(state).await { + warn!("Failed to set status on eval strategy: {e:?}"); + } - if state == hubcaps::statuses::State::Success { + if state == StatusState::Success { Ok(()) } else { Err(()) @@ -439,14 +476,42 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { .all_evaluations_passed(&mut overall_status) .await?; - response.extend(schedule_builds(complete.builds, auto_schedule_build_archs)); + if let (Some(ref queue), Some(ref nix), Some(jobset_id)) = ( + self.hydra_eval_queue.clone(), + self.hydra_eval_nix.clone(), + self.hydra_eval_jobset_id, + ) { + let drv_paths = resolve_attrs_to_drv_paths( + nix, + std::path::Path::new(&refpath), + &complete.builds, + ); + if !drv_paths.is_empty() { + info!( + "Publishing {} drv paths to hydra-eval-jobs for PR #{}", + drv_paths.len(), + job.pr.number + ); + response.push(worker::publish_serde_action( + None, + Some(queue.clone()), + &hydra_eval_job::HydraEvalJob { + repo: job.repo.clone(), + pr: job.pr.clone(), + drv_paths, + request_id: Uuid::new_v4().to_string(), + jobset_id, + }, + )); + } + } overall_status - .set_with_description("^.^!", hubcaps::statuses::State::Success) + .set_with_description("^.^!", StatusState::Success) .await?; } else { overall_status - .set_with_description("Complete, with errors", hubcaps::statuses::State::Failure) + .set_with_description("Complete, with errors", StatusState::Failure) .await?; } @@ -457,100 +522,64 @@ impl<'a, E: stats::SysEvents + 'static> OneEval<'a, E> { } } -fn schedule_builds( - builds: Vec, - auto_schedule_build_archs: Vec, -) -> Vec { - let mut response = vec![]; - info!( - "Scheduling build jobs {:?} on arches {:?}", - builds, auto_schedule_build_archs - ); - for buildjob in builds { - for arch in auto_schedule_build_archs.iter() { - let (exchange, routingkey) = arch.as_build_destination(); - response.push(worker::publish_serde_action( - exchange, routingkey, &buildjob, - )); - } - response.push(worker::publish_serde_action( - Some("build-results".to_string()), - None, - &buildjob::QueuedBuildJobs { - job: buildjob, - architectures: auto_schedule_build_archs - .iter() - .map(|arch| arch.to_string()) - .collect(), - }, - )); +fn resolve_attrs_to_drv_paths( + nix: &nix::Nix, + nixpkgs: &std::path::Path, + builds: &[buildjob::BuildJob], +) -> Vec { + let mut all_attrs: Vec = builds.iter().flat_map(|b| b.attrs.clone()).collect(); + all_attrs.sort(); + all_attrs.dedup(); + + if all_attrs.is_empty() { + return vec![]; } - response -} - -pub async fn update_labels( - issueref: &hubcaps::issues::IssueRef, - add: &[String], - remove: &[String], -) { - let l = issueref.labels(); - let issue = issueref.get().await.expect("Failed to get issue"); - - let existing: Vec = issue.labels.iter().map(|l| l.name.clone()).collect(); - - let to_add: Vec<&str> = add - .iter() - .filter(|l| !existing.contains(l)) // Remove labels already on the issue - .map(|l| l.as_ref()) - .collect(); - - let to_remove: Vec = remove - .iter() - .filter(|l| existing.contains(l)) // Remove labels already on the issue - .cloned() - .collect(); - - let issue = issue.number; - - info!("Labeling issue #{issue}: + {to_add:?} , - {to_remove:?}, = {existing:?}"); - - l.add(to_add.clone()) - .await - .unwrap_or_else(|err| panic!("Failed to add labels {to_add:?} to issue #{issue}: {err:?}")); - - for label in to_remove { - l.remove(&label).await.unwrap_or_else(|err| { - panic!("Failed to remove label {label:?} from issue #{issue}: {err:?}") - }); + let file = builds + .first() + .and_then(|b| b.subset.clone()) + .map(|s| match s { + crate::commentparser::Subset::NixOS => nix::File::ReleaseNixOS, + crate::commentparser::Subset::Nixpkgs => nix::File::DefaultNixpkgs, + }) + .unwrap_or(nix::File::DefaultNixpkgs); + + // Try batch instantiation first for performance (single nix-instantiate process) + match nix.safely_instantiate_attrs(nixpkgs, file, all_attrs.clone()) { + Ok(f) => { + return BufReader::new(f) + .lines() + .map_while(Result::ok) + .filter(|line| line.trim().ends_with(".drv")) + .map(|line| line.trim().to_owned()) + .collect(); + } + Err(_) => warn!("Batch instantiation failed, falling back to per-attr fallback"), } -} - -fn issue_is_wip(issue: &hubcaps::issues::Issue) -> bool { - issue.title.starts_with("WIP:") || issue.title.contains("[WIP]") -} -/// Determine whether or not to use the "old" status prefix, `grahamcofborg`, or -/// the new one, `ofborg`. -/// -/// If the PR already has any `grahamcofborg`-prefixed statuses, continue to use -/// that (e.g. if someone used `@ofborg eval`, `@ofborg build`, `@ofborg test`). -/// Otherwise, if it's a new PR or was recently force-pushed (and therefore -/// doesn't have any old `grahamcofborg`-prefixed statuses), use the new prefix. -pub async fn get_prefix( - statuses: hubcaps::statuses::Statuses, - sha: &str, -) -> Result<&str, CommitStatusError> { - if statuses - .list(sha) - .await? - .iter() - .any(|s| s.context.starts_with("grahamcofborg-")) - { - Ok("grahamcofborg") - } else { - Ok("ofborg") - } + // Fallback: try each attr individually + all_attrs + .into_iter() + .flat_map( + |attr| match nix.safely_instantiate_attrs(nixpkgs, file, vec![attr.clone()]) { + Ok(f) => BufReader::new(f) + .lines() + .map_while(Result::ok) + .filter(|line| line.trim().ends_with(".drv")) + .map(|line| line.trim().to_owned()) + .collect::>(), + Err(f) => { + let stderr: Vec = + BufReader::new(f).lines().map_while(Result::ok).collect(); + warn!( + "nix-instantiate failed for attr '{attr}': {:?}", + stderr.join("\n") + ); + vec![] + } + }, + ) + .collect() } enum EvalWorkerError { diff --git a/ofborg/src/tasks/evaluationfilter.rs b/ofborg/src/tasks/evaluationfilter.rs index 91ca4f49..07d5881d 100644 --- a/ofborg/src/tasks/evaluationfilter.rs +++ b/ofborg/src/tasks/evaluationfilter.rs @@ -1,9 +1,7 @@ -use crate::acl; -use crate::ghevent; -use crate::message::{Pr, Repo, evaluationjob}; -use crate::worker; +use tracing::{Instrument, debug_span, info}; -use tracing::{debug_span, info}; +use crate::message::{Pr, Repo, evaluationjob}; +use crate::{acl, ghevent, worker}; pub struct EvaluationFilterWorker { acl: acl::Acl, @@ -35,70 +33,76 @@ impl worker::SimpleWorker for EvaluationFilterWorker { async fn consumer(&mut self, job: &ghevent::PullRequestEvent) -> worker::Actions { let span = debug_span!("job", pr = ?job.number); - let _enter = span.enter(); - - if !self.acl.is_repo_eligible(&job.repository.full_name) { - info!("Repo not authorized ({})", job.repository.full_name); - return vec![worker::Action::Ack]; - } + async { + if !self.acl.is_repo_eligible(&job.repository.full_name) { + info!("Repo not authorized ({})", job.repository.full_name); + return vec![worker::Action::Ack]; + } - if job.pull_request.state != ghevent::PullRequestState::Open { - info!( - "PR is not open ({}#{})", - job.repository.full_name, job.number - ); - return vec![worker::Action::Ack]; - } + if job.pull_request.state != ghevent::PullRequestState::Open { + info!( + "PR is not open ({}#{})", + job.repository.full_name, job.number + ); + return vec![worker::Action::Ack]; + } - let interesting: bool = match job.action { - ghevent::PullRequestAction::Opened => true, - ghevent::PullRequestAction::Synchronize => true, - ghevent::PullRequestAction::Reopened => true, - ghevent::PullRequestAction::Edited => { - if let Some(ref changes) = job.changes { - changes.base.is_some() - } else { - false + let interesting: bool = match job.action { + ghevent::PullRequestAction::Opened => true, + ghevent::PullRequestAction::Synchronize => true, + ghevent::PullRequestAction::Reopened => true, + ghevent::PullRequestAction::Edited => { + if let Some(ref changes) = job.changes { + changes.base.is_some() + } else { + false + } } + _ => false, + }; + + if !interesting { + info!( + "Not interesting: {}#{} because of {:?}", + job.repository.full_name, job.number, job.action + ); + + return vec![worker::Action::Ack]; } - _ => false, - }; - if !interesting { info!( - "Not interesting: {}#{} because of {:?}", + "Found {}#{} to be interesting because of {:?}", job.repository.full_name, job.number, job.action ); + let repo_msg = Repo { + clone_url: job.repository.clone_url.clone(), + full_name: job.repository.full_name.clone(), + owner: job.repository.owner.login.clone(), + name: job.repository.name.clone(), + }; + + let pr_msg = Pr { + number: job.number, + head_sha: job.pull_request.head.sha.clone(), + target_branch: Some(job.pull_request.base.git_ref.clone()), + }; + + let msg = evaluationjob::EvaluationJob { + repo: repo_msg, + pr: pr_msg, + }; - return vec![worker::Action::Ack]; + vec![ + worker::publish_serde_action( + None, + Some("mass-rebuild-check-jobs".to_owned()), + &msg, + ), + worker::Action::Ack, + ] } - - info!( - "Found {}#{} to be interesting because of {:?}", - job.repository.full_name, job.number, job.action - ); - let repo_msg = Repo { - clone_url: job.repository.clone_url.clone(), - full_name: job.repository.full_name.clone(), - owner: job.repository.owner.login.clone(), - name: job.repository.name.clone(), - }; - - let pr_msg = Pr { - number: job.number, - head_sha: job.pull_request.head.sha.clone(), - target_branch: Some(job.pull_request.base.git_ref.clone()), - }; - - let msg = evaluationjob::EvaluationJob { - repo: repo_msg, - pr: pr_msg, - }; - - vec![ - worker::publish_serde_action(None, Some("mass-rebuild-check-jobs".to_owned()), &msg), - worker::Action::Ack, - ] + .instrument(span) + .await } } diff --git a/ofborg/src/tasks/githubcommentfilter.rs b/ofborg/src/tasks/githubcommentfilter.rs index 0656b1d4..decbc5a5 100644 --- a/ofborg/src/tasks/githubcommentfilter.rs +++ b/ofborg/src/tasks/githubcommentfilter.rs @@ -1,19 +1,18 @@ -use crate::acl; -use crate::commentparser; -use crate::ghevent; -use crate::message::{Pr, Repo, buildjob, evaluationjob}; -use crate::worker; - -use tracing::{debug_span, error, info}; +use octocrab::Octocrab; +use tracing::{Instrument, debug_span, error, info}; use uuid::Uuid; +use crate::github::GithubRepo; +use crate::message::{Pr, Repo, buildjob, evaluationjob}; +use crate::{acl, commentparser, ghevent, worker}; + pub struct GitHubCommentWorker { acl: acl::Acl, - github: hubcaps::Github, + github: Octocrab, } impl GitHubCommentWorker { - pub fn new(acl: acl::Acl, github: hubcaps::Github) -> GitHubCommentWorker { + pub fn new(acl: acl::Acl, github: Octocrab) -> GitHubCommentWorker { GitHubCommentWorker { acl, github } } } @@ -43,129 +42,127 @@ impl worker::SimpleWorker for GitHubCommentWorker { #[allow(clippy::cognitive_complexity)] async fn consumer(&mut self, job: &ghevent::IssueComment) -> worker::Actions { let span = debug_span!("job", pr = ?job.issue.number); - let _enter = span.enter(); - - if job.action == ghevent::IssueCommentAction::Deleted - || job.action == ghevent::IssueCommentAction::Pinned - || job.action == ghevent::IssueCommentAction::Unpinned - { - return vec![worker::Action::Ack]; - } + async { + if job.action == ghevent::IssueCommentAction::Deleted + || job.action == ghevent::IssueCommentAction::Pinned + || job.action == ghevent::IssueCommentAction::Unpinned + { + return vec![worker::Action::Ack]; + } - let instructions = commentparser::parse(&job.comment.body); - if instructions.is_none() { - return vec![worker::Action::Ack]; - } + let instructions = commentparser::parse(&job.comment.body); + if instructions.is_none() { + return vec![worker::Action::Ack]; + } - let build_destinations = self.acl.build_job_architectures_for_user_repo( - &job.comment.user.login, - &job.repository.full_name, - ); + let build_destinations = self.acl.build_job_architectures_for_user_repo( + &job.comment.user.login, + &job.repository.full_name, + ); - if build_destinations.is_empty() { - info!("No build destinations for: {:?}", job); - // Don't process comments if they can't build anything - return vec![worker::Action::Ack]; - } + if build_destinations.is_empty() { + info!("No build destinations for: {:?}", job); + // Don't process comments if they can't build anything + return vec![worker::Action::Ack]; + } - info!("Got job: {:?}", job); - - let instructions = commentparser::parse(&job.comment.body); - info!("Instructions: {:?}", instructions); - - let pr = self - .github - .repo( - job.repository.owner.login.clone(), - job.repository.name.clone(), - ) - .pulls() - .get(job.issue.number) - .get() - .await; - - if let Err(x) = pr { - info!( - "fetching PR {}#{} from GitHub yielded error {}", - job.repository.full_name, job.issue.number, x - ); - return vec![worker::Action::Ack]; - } + info!("Got job: {:?}", job); - let pr = pr.unwrap(); - - let repo_msg = Repo { - clone_url: job.repository.clone_url.clone(), - full_name: job.repository.full_name.clone(), - owner: job.repository.owner.login.clone(), - name: job.repository.name.clone(), - }; - - let pr_msg = Pr { - number: job.issue.number, - head_sha: pr.head.sha.clone(), - target_branch: Some(pr.base.commit_ref), - }; - - let mut response: Vec = vec![]; - if let Some(instructions) = instructions { - for instruction in instructions { - match instruction { - commentparser::Instruction::Build(subset, attrs) => { - let build_destinations = match subset { - commentparser::Subset::NixOS => build_destinations - .clone() - .into_iter() - .filter(|x| x.can_run_nixos_tests()) - .collect(), - _ => build_destinations.clone(), - }; - - let msg = buildjob::BuildJob::new( - repo_msg.clone(), - pr_msg.clone(), - subset, - attrs, - None, - None, - Uuid::new_v4().to_string(), - ); - - for arch in build_destinations.iter() { - let (exchange, routingkey) = arch.as_build_destination(); - response.push(worker::publish_serde_action(exchange, routingkey, &msg)); - } + let instructions = commentparser::parse(&job.comment.body); + info!("Instructions: {:?}", instructions); - response.push(worker::publish_serde_action( - Some("build-results".to_string()), - None, - &buildjob::QueuedBuildJobs { - job: msg, - architectures: build_destinations - .iter() - .cloned() - .map(|arch| arch.to_string()) + let github_repo = GithubRepo::new( + self.github.clone(), + &job.repository.owner.login, + &job.repository.name, + ); + let pr = match github_repo.pulls().get(job.issue.number).await { + Ok(pr) => pr, + Err(x) => { + info!( + "fetching PR {}#{} from GitHub yielded error: {}", + job.repository.full_name, job.issue.number, x + ); + return vec![worker::Action::Ack]; + } + }; + + let repo_msg = Repo { + clone_url: job.repository.clone_url.clone(), + full_name: job.repository.full_name.clone(), + owner: job.repository.owner.login.clone(), + name: job.repository.name.clone(), + }; + + let pr_msg = Pr { + number: job.issue.number, + head_sha: pr.head.sha.clone(), + target_branch: Some(pr.base.ref_field.clone()), + }; + + let mut response: Vec = vec![]; + if let Some(instructions) = instructions { + for instruction in instructions { + match instruction { + commentparser::Instruction::Build(subset, attrs) => { + let build_destinations = match subset { + commentparser::Subset::NixOS => build_destinations + .clone() + .into_iter() + .filter(|x| x.can_run_nixos_tests()) .collect(), - }, - )); - } - commentparser::Instruction::Eval => { - let msg = evaluationjob::EvaluationJob { - repo: repo_msg.clone(), - pr: pr_msg.clone(), - }; - - response.push(worker::publish_serde_action( - None, - Some("mass-rebuild-check-jobs".to_owned()), - &msg, - )); + _ => build_destinations.clone(), + }; + + let msg = buildjob::BuildJob::new( + repo_msg.clone(), + pr_msg.clone(), + subset, + attrs, + None, + None, + Uuid::new_v4().to_string(), + ); + + for arch in build_destinations.iter() { + let (exchange, routingkey) = arch.as_build_destination(); + response + .push(worker::publish_serde_action(exchange, routingkey, &msg)); + } + + response.push(worker::publish_serde_action( + Some("build-results".to_string()), + None, + &buildjob::QueuedBuildJobs { + job: msg, + architectures: build_destinations + .iter() + .cloned() + .map(|arch| arch.to_string()) + .collect(), + }, + )); + } + commentparser::Instruction::Eval => { + let msg = evaluationjob::EvaluationJob { + repo: repo_msg.clone(), + pr: pr_msg.clone(), + }; + + response.push(worker::publish_serde_action( + None, + Some("mass-rebuild-check-jobs".to_owned()), + &msg, + )); + } } } } - } - response.push(worker::Action::Ack); - response + response.push(worker::Action::Ack); + response + } + .instrument(span) + .await } } diff --git a/ofborg/src/tasks/githubcommentposter.rs b/ofborg/src/tasks/githubcommentposter.rs index 12c7c7fd..3a137897 100644 --- a/ofborg/src/tasks/githubcommentposter.rs +++ b/ofborg/src/tasks/githubcommentposter.rs @@ -1,13 +1,14 @@ +use chrono::Utc; +use octocrab::params::checks::{CheckRunConclusion, CheckRunOutput, CheckRunStatus}; +use tracing::{Instrument, debug_span, info, warn}; + use crate::config::GithubAppVendingMachine; +use crate::github::GithubRepo; use crate::message::Repo; use crate::message::buildjob::{BuildJob, QueuedBuildJobs}; use crate::message::buildresult::{BuildResult, BuildStatus, LegacyBuildResult}; use crate::worker; -use chrono::{DateTime, Utc}; -use hubcaps::checks::{CheckRunOptions, CheckRunState, Conclusion, Output}; -use tracing::{debug, debug_span, info, warn}; - pub struct GitHubCommentPoster { github_vend: GithubAppVendingMachine, } @@ -18,6 +19,15 @@ impl GitHubCommentPoster { } } +pub struct CheckRunInfo { + pub name: String, + pub details_url: String, + pub output: CheckRunOutput, + pub conclusion: Option, + pub status: Option, + pub completed_at: Option>, +} + pub enum PostableEvent { BuildQueued(QueuedBuildJobs), BuildFinished(BuildResult), @@ -52,58 +62,82 @@ impl worker::SimpleWorker for GitHubCommentPoster { } async fn consumer(&mut self, job: &PostableEvent) -> worker::Actions { - let mut checks: Vec = vec![]; + let mut check_runs: Vec = vec![]; let repo: Repo; + let pr_number: u64; + let head_sha: String; - let pr = match job { + match job { PostableEvent::BuildQueued(queued_job) => { repo = queued_job.job.repo.clone(); + pr_number = queued_job.job.pr.number; + head_sha = queued_job.job.pr.head_sha.clone(); for architecture in queued_job.architectures.iter() { - checks.push(job_to_check(&queued_job.job, architecture, Utc::now())); + check_runs.push(job_to_check_info(&queued_job.job, architecture)); } - queued_job.job.pr.to_owned() } PostableEvent::BuildFinished(finished_job) => { let result = finished_job.legacy(); repo = result.repo.clone(); - checks.push(result_to_check(&result, Utc::now())); - finished_job.pr() + pr_number = result.pr.number; + head_sha = result.pr.head_sha.clone(); + check_runs.push(result_to_check_info(&result)); } }; - let span = debug_span!("job", pr = ?pr.number); - let _enter = span.enter(); - - for check in checks { - info!( - "check {:?} {} {}", - check.status, - check.name, - check.details_url.as_ref().unwrap_or(&String::from("-")) - ); - debug!("{:?}", check); - - let check_create_attempt = self - .github_vend - .for_repo(&repo.owner, &repo.name) - .await - .unwrap() - .repo(repo.owner.clone(), repo.name.clone()) - .checkruns() - .create(&check) - .await; - - match check_create_attempt { - Ok(_) => info!("Successfully sent."), - Err(err) => warn!("Failed to send check {:?}", err), + let span = debug_span!("job", pr = ?pr_number); + async { + let octocrab_ref = match self.github_vend.for_repo(&repo.owner, &repo.name).await { + Some(client) => client.clone(), + None => { + warn!( + "No GitHub installation found for {}/{}, skipping checks", + repo.owner, repo.name + ); + return vec![worker::Action::Ack]; + } + }; + + for check in check_runs { + info!( + "check {:?} {} {}", + check.status, check.name, check.details_url, + ); + + let github_repo = GithubRepo::new(octocrab_ref.clone(), &repo.owner, &repo.name); + let checks_handler = github_repo.checks(); + let mut builder = checks_handler + .create_check_run(check.name, head_sha.clone()) + .details_url(check.details_url); + + builder = builder.output(check.output); + + if let Some(completed_at) = check.completed_at { + builder = builder.completed_at(completed_at); + } + + if let Some(conclusion) = check.conclusion { + builder = builder.conclusion(conclusion); + } + + if let Some(status) = check.status { + builder = builder.status(status); + } + + match builder.send().await { + Ok(_) => info!("Successfully sent check."), + Err(err) => warn!("Failed to send check {:?}", err), + } } - } - vec![worker::Action::Ack] + vec![worker::Action::Ack] + } + .instrument(span) + .await } } -fn job_to_check(job: &BuildJob, architecture: &str, timestamp: DateTime) -> CheckRunOptions { +fn job_to_check_info(job: &BuildJob, architecture: &str) -> CheckRunInfo { let mut all_attrs: Vec = job.attrs.clone(); all_attrs.sort(); @@ -111,26 +145,31 @@ fn job_to_check(job: &BuildJob, architecture: &str, timestamp: DateTime) -> all_attrs = vec![String::from("(unknown attributes)")]; } - CheckRunOptions { - name: format!("{} on {architecture}", all_attrs.join(", ")), - actions: None, - completed_at: None, - started_at: Some(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), + let name = format!("{} on {architecture}", all_attrs.join(", ")); + let details_url = format!( + "https://logs.ofborg.org/?key={}/{}.{}", + &job.repo.owner.to_lowercase(), + &job.repo.name.to_lowercase(), + job.pr.number, + ); + + CheckRunInfo { + name, + details_url, + output: CheckRunOutput { + title: "Queued".to_string(), + summary: String::new(), + text: None, + annotations: vec![], + images: vec![], + }, conclusion: None, - details_url: Some(format!( - "https://logs.ofborg.org/?key={}/{}.{}", - &job.repo.owner.to_lowercase(), - &job.repo.name.to_lowercase(), - job.pr.number, - )), - external_id: None, - head_sha: job.pr.head_sha.clone(), - output: None, - status: Some(CheckRunState::Queued), + status: Some(CheckRunStatus::Queued), + completed_at: None, } } -fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> CheckRunOptions { +fn result_to_check_info(result: &LegacyBuildResult) -> CheckRunInfo { let mut all_attrs: Vec = vec![result.attempted_attrs.clone(), result.skipped_attrs.clone()] .into_iter() @@ -143,7 +182,14 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> Chec all_attrs = vec![String::from("(unknown attributes)")]; } - let conclusion: Conclusion = result.status.clone().into(); + let conclusion: CheckRunConclusion = match result.status { + BuildStatus::Skipped => CheckRunConclusion::Skipped, + BuildStatus::Success => CheckRunConclusion::Success, + BuildStatus::Failure => CheckRunConclusion::Neutral, + BuildStatus::TimedOut => CheckRunConclusion::Neutral, + BuildStatus::UnexpectedError { .. } => CheckRunConclusion::Neutral, + BuildStatus::HashMismatch => CheckRunConclusion::Failure, + }; let mut summary: Vec = vec![]; if let Some(ref attempted) = result.attempted_attrs { @@ -164,8 +210,6 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> Chec )); } - // Allow the clippy violation for improved readability - #[allow(clippy::vec_init_then_push)] let text: String = if !result.output.is_empty() { let mut reply: Vec = vec![]; @@ -180,30 +224,28 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime) -> Chec String::from("No partial log is available.") }; - CheckRunOptions { - name: format!("{} on {}", all_attrs.join(", "), result.system), - actions: None, - completed_at: Some(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), - started_at: None, - conclusion: Some(conclusion), - details_url: Some(format!( - "https://logs.ofborg.org/?key={}/{}.{}&attempt_id={}", - &result.repo.owner.to_lowercase(), - &result.repo.name.to_lowercase(), - result.pr.number, - result.attempt_id, - )), - external_id: Some(result.attempt_id.clone()), - head_sha: result.pr.head_sha.clone(), - - output: Some(Output { - annotations: None, - images: None, + let name = format!("{} on {}", all_attrs.join(", "), result.system); + let details_url = format!( + "https://logs.ofborg.org/?key={}/{}.{}&attempt_id={}", + &result.repo.owner.to_lowercase(), + &result.repo.name.to_lowercase(), + result.pr.number, + result.attempt_id, + ); + + CheckRunInfo { + name, + details_url, + output: CheckRunOutput { + title: result.status.clone().into(), summary: summary.join("\n"), text: Some(text), - title: result.status.clone().into(), - }), - status: Some(CheckRunState::Completed), + annotations: vec![], + images: vec![], + }, + conclusion: Some(conclusion), + status: Some(CheckRunStatus::Completed), + completed_at: Some(Utc::now()), } } @@ -221,545 +263,343 @@ fn list_segment(name: &str, things: &[String]) -> Vec { #[cfg(test)] mod tests { use super::*; + use crate::message::buildjob::BuildJob; use crate::message::{Pr, Repo}; - use chrono::TimeZone; + + fn check_passing_log() -> Vec { + vec![ + "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'" + .to_owned(), + "make[2]: Nothing to be done for 'install'.".to_owned(), + "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'" + .to_owned(), + "make[1]: Nothing to be done for 'install-target'.".to_owned(), + "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), + "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'" + .to_owned(), + "post-installation fixup".to_owned(), + "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip" + .to_owned(), + "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1" + .to_owned(), + "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), + ] + } + + fn partial_log_text(lines: &[String]) -> String { + let mut reply: Vec = vec![]; + reply.push("## Partial log".to_owned()); + reply.push("".to_owned()); + reply.push("```".to_owned()); + reply.extend(lines.iter().cloned()); + reply.push("```".to_owned()); + reply.join("\n") + } + + fn base_repo() -> Repo { + Repo { + clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), + full_name: "NixOS/nixpkgs".to_owned(), + owner: "NixOS".to_owned(), + name: "nixpkgs".to_owned(), + } + } + + fn base_pr() -> Pr { + Pr { + head_sha: "abc123".to_owned(), + number: 2345, + target_branch: Some("master".to_owned()), + } + } + + fn base_result() -> LegacyBuildResult { + LegacyBuildResult { + repo: base_repo(), + pr: base_pr(), + output: vec![], + attempt_id: "neatattemptid".to_owned(), + request_id: "bogus-request-id".to_owned(), + system: "x86_64-linux".to_owned(), + attempted_attrs: None, + skipped_attrs: None, + status: BuildStatus::Skipped, + } + } #[test] pub fn test_queued_build() { let job = BuildJob { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, + repo: base_repo(), + pr: base_pr(), logs: None, statusreport: None, subset: None, - request_id: "bogus-request-id".to_owned(), attrs: vec!["foo".to_owned(), "bar".to_owned()], }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); + let result = job_to_check_info(&job, "x86_64-linux"); + assert_eq!(result.name, "bar, foo on x86_64-linux"); assert_eq!( - job_to_check(&job, "x86_64-linux", timestamp), - CheckRunOptions { - name: "bar, foo on x86_64-linux".to_string(), - actions: None, - started_at: Some("2023-04-20T13:37:42Z".to_string()), - completed_at: None, - status: Some(CheckRunState::Queued), - conclusion: None, - details_url: Some("https://logs.ofborg.org/?key=nixos/nixpkgs.2345".to_string()), - external_id: None, - head_sha: "abc123".to_string(), - output: None, - } + result.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345" ); + assert_eq!(result.output.title, "Queued"); + assert_eq!(result.output.summary, ""); + assert_eq!(result.output.text, None); + assert!(result.output.annotations.is_empty()); + assert!(result.output.images.is_empty()); + assert!(result.conclusion.is_none()); + assert!(matches!(result.status, Some(CheckRunStatus::Queued))); } #[test] pub fn test_check_passing_build() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: Some(vec!["bar".to_owned()]), status: BuildStatus::Success, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "bar, foo on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "bar, foo on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Success), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Success".to_string(), - summary: "Attempted: foo - -The following builds were skipped because they don't evaluate on x86_64-linux: bar -" - .to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Success) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Success"); + assert_eq!( + check.output.summary, + concat!( + "Attempted: foo", + "\n", + "\n", + "The following builds were skipped because they don't evaluate on x86_64-linux: bar", + "\n", + ) + ); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_failing_build() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: None, status: BuildStatus::Failure, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "foo on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "foo on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Neutral), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Failure".to_string(), - summary: "Attempted: foo -" - .to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Neutral) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Failure"); + assert_eq!(check.output.summary, "Attempted: foo\n"); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_timedout_build() { + let mut log = check_passing_log(); + log.push( + "building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' \ + timed out after 1 seconds" + .to_owned(), + ); + log.push( + "error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed" + .to_owned(), + ); + let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds".to_owned(), - "error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: log.clone(), attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: None, status: BuildStatus::TimedOut, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "foo on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "foo on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Neutral), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Timed out, unknown build status".to_string(), - summary: "Attempted: foo - -Build timed out." - .to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -building of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' timed out after 1 seconds -error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Neutral) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Timed out, unknown build status"); + assert_eq!( + check.output.summary, + concat!("Attempted: foo", "\n", "\n", "Build timed out.") + ); + assert_eq!(check.output.text, Some(partial_log_text(&log))); } #[test] pub fn test_check_passing_build_unspecified_attributes() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Success, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "(unknown attributes) on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "(unknown attributes) on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Success), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Success".to_string(), - summary: "".to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Success) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Success"); + assert_eq!(check.output.summary, ""); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_failing_build_unspecified_attributes() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, - output: vec![ - "make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[2]: Nothing to be done for 'install'.".to_owned(), - "make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline'".to_owned(), - "make[1]: Nothing to be done for 'install-target'.".to_owned(), - "make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1'".to_owned(), - "removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info'".to_owned(), - "post-installation fixup".to_owned(), - "strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip".to_owned(), - "patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - "/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1".to_owned(), - ], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), + output: check_passing_log(), attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Failure, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "(unknown attributes) on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "(unknown attributes) on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Neutral), - details_url: Some( - "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" - .to_string() - ), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "Failure".to_string(), - summary: "".to_string(), - text: Some( - "## Partial log - -``` -make[2]: Entering directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[2]: Nothing to be done for 'install'. -make[2]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1/readline' -make[1]: Nothing to be done for 'install-target'. -make[1]: Leaving directory '/private/tmp/nix-build-gdb-8.1.drv-0/gdb-8.1' -removed '/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1/share/info/bfd.info' -post-installation fixup -strip is /nix/store/5a88zk3jgimdmzg8rfhvm93kxib3njf9-cctools-binutils-darwin/bin/strip -patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -/nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29-gdb-8.1 -```" - .to_string() - ), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Neutral) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "Failure"); + assert_eq!(check.output.summary, ""); + assert_eq!( + check.output.text, + Some(partial_log_text(&check_passing_log())) ); } #[test] pub fn test_check_no_attempt() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, output: vec!["foo".to_owned()], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), attempted_attrs: None, skipped_attrs: Some(vec!["not-attempted".to_owned()]), status: BuildStatus::Skipped, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "not-attempted on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "not-attempted on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Skipped), - details_url: Some("https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "No attempt".to_string(), - summary: "The following builds were skipped because they don\'t evaluate on x86_64-linux: not-attempted -".to_string(), - text: Some("## Partial log - -``` -foo -```".to_string()), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Skipped) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "No attempt"); + assert_eq!( + check.output.summary, + concat!( + "The following builds were skipped because they don't evaluate on x86_64-linux: \ + not-attempted", + "\n", + ) + ); + assert_eq!( + check.output.text, + Some( + concat!( + "## Partial log", + "\n", + "\n", + "```", + "\n", + "foo", + "\n", + "```" + ) + .to_string() + ) ); } #[test] pub fn test_check_no_attempt_no_log() { let result = LegacyBuildResult { - repo: Repo { - clone_url: "https://github.com/nixos/nixpkgs.git".to_owned(), - full_name: "NixOS/nixpkgs".to_owned(), - owner: "NixOS".to_owned(), - name: "nixpkgs".to_owned(), - }, - pr: Pr { - head_sha: "abc123".to_owned(), - number: 2345, - target_branch: Some("master".to_owned()), - }, output: vec![], - attempt_id: "neatattemptid".to_owned(), - request_id: "bogus-request-id".to_owned(), - system: "x86_64-linux".to_owned(), attempted_attrs: None, skipped_attrs: Some(vec!["not-attempted".to_owned()]), status: BuildStatus::Skipped, + ..base_result() }; - let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); - + let check = result_to_check_info(&result); + assert_eq!(check.name, "not-attempted on x86_64-linux"); assert_eq!( - result_to_check(&result, timestamp), - CheckRunOptions { - name: "not-attempted on x86_64-linux".to_string(), - actions: None, - started_at: None, - completed_at: Some("2023-04-20T13:37:42Z".to_string()), - status: Some(CheckRunState::Completed), - conclusion: Some(Conclusion::Skipped), - details_url: Some("https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid".to_string()), - external_id: Some("neatattemptid".to_string()), - head_sha: "abc123".to_string(), - output: Some(Output { - title: "No attempt".to_string(), - summary: "The following builds were skipped because they don\'t evaluate on x86_64-linux: not-attempted -".to_string(), - text: Some("No partial log is available.".to_string()), - annotations: None, - images: None, - }) - } + check.details_url, + "https://logs.ofborg.org/?key=nixos/nixpkgs.2345&attempt_id=neatattemptid" + ); + assert!(matches!( + check.conclusion, + Some(CheckRunConclusion::Skipped) + )); + assert!(matches!(check.status, Some(CheckRunStatus::Completed))); + assert_eq!(check.output.title, "No attempt"); + assert_eq!( + check.output.summary, + concat!( + "The following builds were skipped because they don't evaluate on x86_64-linux: \ + not-attempted", + "\n", + ) + ); + assert_eq!( + check.output.text, + Some("No partial log is available.".to_string()) ); } } diff --git a/ofborg/src/tasks/log_message_collector.rs b/ofborg/src/tasks/log_message_collector.rs index f359ffe7..be182642 100644 --- a/ofborg/src/tasks/log_message_collector.rs +++ b/ofborg/src/tasks/log_message_collector.rs @@ -1,8 +1,3 @@ -use crate::message::buildlogmsg::{BuildLogMsg, BuildLogStart}; -use crate::message::buildresult::BuildResult; -use crate::worker; -use crate::writetoline::LineWriter; - use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::path::{Component, Path, PathBuf}; @@ -10,6 +5,11 @@ use std::path::{Component, Path, PathBuf}; use lru_cache::LruCache; use tracing::warn; +use crate::message::buildlogmsg::{BuildLogMsg, BuildLogStart}; +use crate::message::buildresult::BuildResult; +use crate::worker; +use crate::writetoline::LineWriter; + #[derive(Eq, PartialEq, Hash, Debug, Clone)] pub struct LogFrom { routing_key: String, diff --git a/ofborg/src/tasks/mod.rs b/ofborg/src/tasks/mod.rs index 5aab0fa6..e8c292fc 100644 --- a/ofborg/src/tasks/mod.rs +++ b/ofborg/src/tasks/mod.rs @@ -1,4 +1,3 @@ -pub mod build; pub mod eval; pub mod evaluate; pub mod evaluationfilter; diff --git a/ofborg/src/tasks/statscollector.rs b/ofborg/src/tasks/statscollector.rs index fef23ad3..9b2d8067 100644 --- a/ofborg/src/tasks/statscollector.rs +++ b/ofborg/src/tasks/statscollector.rs @@ -1,8 +1,8 @@ +use tracing::error; + use crate::stats; use crate::worker; -use tracing::error; - pub struct StatCollectorWorker { events: E, collector: stats::MetricCollector, diff --git a/ofborg/src/test_utils.rs b/ofborg/src/test_utils.rs new file mode 100644 index 00000000..559f4588 --- /dev/null +++ b/ofborg/src/test_utils.rs @@ -0,0 +1,42 @@ +use async_trait::async_trait; + +#[derive(Debug, Clone)] +pub struct PublishedMessage { + pub exchange: String, + pub routing_key: String, + pub body: Vec, +} + +#[derive(Debug, Default)] +pub struct MockPublisher { + published: tokio::sync::Mutex>, +} + +impl MockPublisher { + pub fn new() -> Self { + Self { + published: tokio::sync::Mutex::new(Vec::new()), + } + } + + pub async fn get_published(&self) -> Vec { + self.published.lock().await.clone() + } + + pub async fn clear(&self) { + self.published.lock().await.clear(); + } +} + +#[async_trait] +impl crate::MessagePublisher for MockPublisher { + async fn publish(&self, exchange: &str, routing_key: &str, body: &[u8]) -> anyhow::Result<()> { + let mut guard = self.published.lock().await; + guard.push(PublishedMessage { + exchange: exchange.to_string(), + routing_key: routing_key.to_string(), + body: body.to_vec(), + }); + Ok(()) + } +} diff --git a/ofborg/src/worker.rs b/ofborg/src/worker.rs index 9569b450..eaae8e3b 100644 --- a/ofborg/src/worker.rs +++ b/ofborg/src/worker.rs @@ -1,7 +1,5 @@ use std::{marker::Send, sync::Arc}; -use serde::Serialize; - pub struct Response {} pub type Actions = Vec; @@ -24,7 +22,7 @@ pub struct QueueMsg { pub content: Vec, } -pub fn publish_serde_action( +pub fn publish_serde_action( exchange: Option, routing_key: Option, msg: &T, @@ -35,7 +33,7 @@ pub fn publish_serde_action( mandatory: false, immediate: false, content_type: Some("application/json".to_owned()), - content: serde_json::to_string(&msg).unwrap().into_bytes(), + content: serde_json::to_vec(&msg).expect("Failed to serialize message for publication"), })) } diff --git a/ofborg/src/writetoline.rs b/ofborg/src/writetoline.rs index 84846424..3ed3f2cf 100644 --- a/ofborg/src/writetoline.rs +++ b/ofborg/src/writetoline.rs @@ -1,5 +1,5 @@ use std::fs::File; -use std::io::{BufRead, BufReader, Seek, SeekFrom, Write}; +use std::io::{BufRead as _, BufReader, Seek, SeekFrom, Write as _}; pub struct LineWriter { file: File,