diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3ba3d0405f..48c21e4ae7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -149,6 +149,7 @@ jobs: run: | if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then export AWS_LC_FIPS_SYS_NO_ASM=1 + git config --global core.longpaths true fi if [[ -z "$CLIPPY_PACKAGES" ]]; then cargo clippy --workspace --all-targets --all-features --keep-going -- -D warnings diff --git a/.github/workflows/test-ffi.yml b/.github/workflows/test-ffi.yml index 994472b9ec..c1bf2ccdc9 100644 --- a/.github/workflows/test-ffi.yml +++ b/.github/workflows/test-ffi.yml @@ -119,6 +119,9 @@ jobs: env: RUSTFLAGS: "${{ matrix.flags }}" run: | + if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then + git config --global core.longpaths true + fi cargo run --bin release --release -- --out $LIBDD_OUTPUT_FOLDER - name: 'Publish libdatadog' uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # 4.6.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8673c56be6..a86f0f138f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,6 +125,9 @@ jobs: - name: "Remove nextest CI report" shell: bash run: rm -rf target/nextest/ci/junit.xml + - name: Configure git long paths (Windows) + if: runner.os == 'Windows' + run: git config --global core.longpaths true - name: "[${{ steps.rust-version.outputs.version}}] cargo build" shell: bash run: | diff --git a/Cargo.lock b/Cargo.lock index f7a8107d58..1b6f99ce46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -397,10 +397,10 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http", + "http 1.1.0", "http-body", "http-body-util", - "itoa", + "itoa 1.0.11", "matchit", "memchr", "mime", @@ -421,7 +421,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", + "http 1.1.0", "http-body", "http-body-util", "mime", @@ -879,8 +879,10 @@ checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.6", ] @@ -1244,7 +1246,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", - "itoa", + "itoa 1.0.11", "ryu", "serde", ] @@ -1418,7 +1420,7 @@ dependencies = [ "bytes", "constcat", "futures", - "http", + "http 1.1.0", "libdd-capabilities", "libdd-capabilities-impl", "libdd-common", @@ -1479,7 +1481,7 @@ dependencies = [ "datadog-live-debugger", "datadog-sidecar-macros", "futures", - "http", + "http 1.1.0", "http-body-util", "httpmock", "libc", @@ -1532,7 +1534,7 @@ dependencies = [ "datadog-ipc", "datadog-live-debugger", "datadog-sidecar", - "http", + "http 1.1.0", "libc", "libdd-common", "libdd-common-ffi", @@ -1610,6 +1612,15 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "derp" +version = "0.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b84cfd9b6fa437e498215e5625e9e3ae3bf9bb54d623028a181c40820db169" +dependencies = [ + "untrusted 0.7.1", +] + [[package]] name = "diff" version = "0.1.13" @@ -2115,7 +2126,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.1.0", "indexmap 2.12.1", "slab", "tokio", @@ -2203,7 +2214,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 1.1.0", "httpdate", "mime", "sha1", @@ -2215,7 +2226,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.1.0", ] [[package]] @@ -2294,6 +2305,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.11", +] + [[package]] name = "http" version = "1.1.0" @@ -2302,7 +2324,7 @@ checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 1.0.11", ] [[package]] @@ -2312,7 +2334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -2323,7 +2345,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", + "http 1.1.0", "http-body", "pin-project-lite", ] @@ -2357,7 +2379,7 @@ dependencies = [ "futures-timer", "futures-util", "headers", - "http", + "http 1.1.0", "http-body-util", "hyper", "hyper-util", @@ -2392,11 +2414,11 @@ dependencies = [ "futures-channel", "futures-util", "h2", - "http", + "http 1.1.0", "http-body", "httparse", "httpdate", - "itoa", + "itoa 1.0.11", "pin-project-lite", "smallvec", "tokio", @@ -2409,7 +2431,7 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", + "http 1.1.0", "hyper", "hyper-util", "rustls", @@ -2444,7 +2466,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", + "http 1.1.0", "http-body", "hyper", "ipnet", @@ -2713,6 +2735,12 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + [[package]] name = "itoa" version = "1.0.11" @@ -2827,7 +2855,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", + "http 1.1.0", "thiserror 1.0.68", ] @@ -2837,7 +2865,7 @@ version = "3.0.0" dependencies = [ "anyhow", "bytes", - "http", + "http 1.1.0", "http-body-util", "libdd-capabilities", "libdd-common", @@ -2858,7 +2886,7 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", + "http 1.1.0", "http-body", "http-body-util", "httparse", @@ -2919,7 +2947,7 @@ dependencies = [ "cxx-build", "errno", "goblin", - "http", + "http 1.1.0", "libc", "libdd-capabilities", "libdd-capabilities-impl", @@ -2978,7 +3006,7 @@ dependencies = [ "either", "futures", "getrandom 0.2.15", - "http", + "http 1.1.0", "http-body-util", "httpmock", "libdd-capabilities", @@ -3057,7 +3085,7 @@ dependencies = [ "anyhow", "async-trait", "cadence", - "http", + "http 1.1.0", "libdd-common", "libdd-shared-runtime", "serde", @@ -3073,7 +3101,7 @@ dependencies = [ "chrono", "derive_more", "faststr", - "http", + "http 1.1.0", "httpmock", "libdd-capabilities", "libdd-capabilities-impl", @@ -3242,7 +3270,7 @@ dependencies = [ "cxx-build", "futures", "hashbrown 0.16.1", - "http", + "http 1.1.0", "http-body-util", "httparse", "indexmap 2.12.1", @@ -3357,10 +3385,11 @@ dependencies = [ "anyhow", "base64 0.22.1", "bytes", + "chrono", "futures", "futures-util", "hashbrown 0.15.1", - "http", + "http 1.1.0", "http-body-util", "hyper", "hyper-util", @@ -3370,6 +3399,8 @@ dependencies = [ "libdd-remote-config", "libdd-trace-protobuf", "manual_future", + "prost", + "rand 0.8.5", "serde", "serde_json", "serde_with", @@ -3381,6 +3412,7 @@ dependencies = [ "tokio", "tokio-util", "tracing", + "tuf", "uuid", ] @@ -3432,7 +3464,7 @@ dependencies = [ "futures", "getrandom 0.2.15", "hashbrown 0.15.1", - "http", + "http 1.1.0", "httpmock", "libc", "libdd-capabilities", @@ -3537,7 +3569,7 @@ dependencies = [ "criterion", "futures", "hashbrown 0.15.1", - "http", + "http 1.1.0", "httpmock", "libdd-capabilities", "libdd-capabilities-impl", @@ -3573,13 +3605,13 @@ dependencies = [ "futures", "getrandom 0.2.15", "hex", - "http", + "http 1.1.0", "http-body", "http-body-util", "httpmock", "hyper", "indexmap 2.12.1", - "itoa", + "itoa 1.0.11", "libdd-capabilities", "libdd-capabilities-impl", "libdd-common", @@ -3610,7 +3642,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", - "http", + "http 1.1.0", "httpmock", "libdd-capabilities-impl", "libdd-common", @@ -3887,7 +3919,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.1.0", "httparse", "memchr", "mime", @@ -4885,7 +4917,7 @@ dependencies = [ "futures-core", "futures-util", "hickory-resolver", - "http", + "http 1.1.0", "http-body", "http-body-util", "hyper", @@ -4928,7 +4960,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.15", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -5092,7 +5124,7 @@ dependencies = [ "aws-lc-rs", "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -5345,7 +5377,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "indexmap 2.12.1", - "itoa", + "itoa 1.0.11", "memchr", "ryu", "serde", @@ -5407,7 +5439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ "indexmap 2.12.1", - "itoa", + "itoa 1.0.11", "ryu", "serde", "unsafe-libyaml", @@ -5666,7 +5698,7 @@ version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fe17b8deb33a9441280b4266c2d257e166bafbaea6e66b4b34ca139c91766d9" dependencies = [ - "itoa", + "itoa 1.0.11", "ryu", "sval", ] @@ -5677,7 +5709,7 @@ version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "854addb048a5bafb1f496c98e0ab5b9b581c3843f03ca07c034ae110d3b7c623" dependencies = [ - "itoa", + "itoa 1.0.11", "ryu", "sval", ] @@ -5967,7 +5999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", - "itoa", + "itoa 1.0.11", "num-conv", "powerfmt", "serde", @@ -6156,7 +6188,7 @@ dependencies = [ "base64 0.22.1", "bytes", "h2", - "http", + "http 1.1.0", "http-body", "http-body-util", "hyper", @@ -6227,7 +6259,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http", + "http 1.1.0", "http-body", "iri-string", "pin-project-lite", @@ -6340,6 +6372,30 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tuf" +version = "0.3.0-beta10" +source = "git+https://github.com/DataDog/rust-tuf/?rev=eb129ccad320b11e8bf99d2f0ff2c415a0795ccb#eb129ccad320b11e8bf99d2f0ff2c415a0795ccb" +dependencies = [ + "chrono", + "data-encoding", + "derp", + "futures-io", + "futures-util", + "http 0.2.12", + "itoa 0.4.8", + "log", + "percent-encoding", + "ring", + "serde", + "serde_derive", + "serde_json", + "tempfile", + "thiserror 1.0.68", + "untrusted 0.7.1", + "url", +] + [[package]] name = "twox-hash" version = "1.6.3" @@ -6398,6 +6454,12 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -6500,7 +6562,7 @@ checksum = "9170e001f458781e92711d2ad666110f153e4e50bfd5cbd02db6547625714187" dependencies = [ "float-cmp", "halfbrown", - "itoa", + "itoa 1.0.11", "ryu", ] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 7c69816d4f..7820d98ea9 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -119,6 +119,7 @@ deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt , Manish Goregaokar , Andre Bogus , Corey Farwell " derive_more,https://github.com/JelteF/derive_more,MIT,Jelte Fennema derive_more-impl,https://github.com/JelteF/derive_more,MIT,Jelte Fennema +derp,https://github.com/heartsucker/derp,ISC,heartsucker diff,https://github.com/utkarshkukreti/diff.rs,MIT OR Apache-2.0,Utkarsh Kukreti digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers dispatch2,https://github.com/madsmtm/objc2,Zlib OR Apache-2.0 OR MIT,"Mads Marquart , Mary " @@ -465,6 +466,7 @@ tracing-log,https://github.com/tokio-rs/tracing,MIT,Tokio Contributors tracing-subscriber,https://github.com/tokio-rs/tracing,MIT,"Eliza Weisman , David Barsky , Tokio Contributors " try-lock,https://github.com/seanmonstar/try-lock,MIT,Sean McArthur +tuf,https://github.com/theupdateframework/rust-tuf,MIT OR Apache-2.0,"heartsucker , Erick Tryzelaar " twox-hash,https://github.com/shepmaster/twox-hash,MIT,Jake Goulding typeid,https://github.com/dtolnay/typeid,MIT OR Apache-2.0,David Tolnay typenum,https://github.com/paholg/typenum,MIT OR Apache-2.0,"Paho Lurie-Gregg , Andre Bogus " diff --git a/datadog-sidecar-ffi/src/lib.rs b/datadog-sidecar-ffi/src/lib.rs index 9fe44880c8..bf81490184 100644 --- a/datadog-sidecar-ffi/src/lib.rs +++ b/datadog-sidecar-ffi/src/lib.rs @@ -266,6 +266,7 @@ pub unsafe extern "C" fn ddog_remote_config_reader_for_endpoint<'a>( language: language.to_utf8_lossy().into(), tracer_version: tracer_version.to_utf8_lossy().into(), endpoint: endpoint.clone(), + agentless: None, }, &Arc::new(Target::new( service_name.to_utf8_lossy().to_string(), diff --git a/datadog-sidecar/src/service/sidecar_server.rs b/datadog-sidecar/src/service/sidecar_server.rs index 8e1ea76937..80a9f82e68 100644 --- a/datadog-sidecar/src/service/sidecar_server.rs +++ b/datadog-sidecar/src/service/sidecar_server.rs @@ -825,6 +825,7 @@ impl SidecarInterface for ConnectionSidecarHandler { language: config.language, tracer_version: config.tracer_version, endpoint: config.endpoint, + agentless: None, }, products: config.remote_config_products, capabilities: config.remote_config_capabilities, diff --git a/datadog-sidecar/src/shm_remote_config.rs b/datadog-sidecar/src/shm_remote_config.rs index 4397f0ef41..5069f0e4e1 100644 --- a/datadog-sidecar/src/shm_remote_config.rs +++ b/datadog-sidecar/src/shm_remote_config.rs @@ -187,7 +187,7 @@ impl FileStorage for ConfigFileStorage { ) -> anyhow::Result> { Ok(Arc::new(StoredShmFile { handle: Mutex::new(Some(store_shm(version, &path, file)?)), - limiter: if path.product == RemoteConfigProduct::LiveDebugging { + limiter: if path.product() == RemoteConfigProduct::LiveDebugging { Some(SHM_LIMITER.lock_or_panic().alloc()) } else { None @@ -294,7 +294,7 @@ impl MultiTargetHandlers ); serialized.push(b'\n'); - if file.refcount.path.product == RemoteConfigProduct::ApmTracing { + if file.refcount.path.product() == RemoteConfigProduct::ApmTracing { let mut handle = file.handle.lock_or_panic(); #[allow(clippy::unwrap_used)] let shm = handle.take().unwrap(); @@ -711,12 +711,7 @@ impl RemoteConfigManager { match read_config(entry.key(), &self.registry) { Ok((parsed, limiter_index)) => { trace!("Adding remote config file {}: {:?}", entry.key(), parsed); - entry.insert(RemoteConfigPath { - source: parsed.source, - product: parsed.product, - config_id: parsed.config_id.clone(), - name: parsed.name.clone(), - }); + entry.insert(parsed.path.clone()); return RemoteConfigUpdate::Add { value: parsed, limiter_index, @@ -784,7 +779,7 @@ impl RemoteConfigManager { /// Can be used to fast-remove configs temporarily. Will be re-applied on next fetch_update(). pub fn unload_configs(&mut self, configs: &[RemoteConfigProduct]) { self.active_configs.retain(|key, path| { - if configs.contains(&path.product) { + if configs.contains(&path.product()) { // self.check_configs should generally be empty here, but be safe if let Some(pos) = self.check_configs.iter().position(|x| x == key) { self.check_configs.swap_remove(pos); @@ -806,29 +801,19 @@ mod tests { tests::dummy_dynamic_config, Configs, DynamicConfigFile, }; use libdd_remote_config::fetch::test_server::RemoteConfigServer; - use libdd_remote_config::{RemoteConfigProduct, RemoteConfigSource}; use manual_future::ManualFuture; use std::sync::LazyLock; - static PATH_FIRST: LazyLock = LazyLock::new(|| RemoteConfigPath { - source: RemoteConfigSource::Employee, - product: RemoteConfigProduct::ApmTracing, - config_id: "1234".to_string(), - name: "config".to_string(), + static PATH_FIRST: LazyLock = LazyLock::new(|| { + RemoteConfigPath::parse("employee/APM_TRACING/1234/config").expect("valid path") }); - static PATH_SECOND: LazyLock = LazyLock::new(|| RemoteConfigPath { - source: RemoteConfigSource::Employee, - product: RemoteConfigProduct::ApmTracing, - config_id: "9876".to_string(), - name: "config".to_string(), + static PATH_SECOND: LazyLock = LazyLock::new(|| { + RemoteConfigPath::parse("employee/APM_TRACING/9876/config").expect("valid path") }); - static PATH_LIVE_DEBUGGER: LazyLock = LazyLock::new(|| RemoteConfigPath { - source: RemoteConfigSource::Employee, - product: RemoteConfigProduct::LiveDebugging, - config_id: "ld-1".to_string(), - name: "config".to_string(), + static PATH_LIVE_DEBUGGER: LazyLock = LazyLock::new(|| { + RemoteConfigPath::parse("employee/LIVE_DEBUGGING/ld-1/config").expect("valid path") }); static DUMMY_TARGET: LazyLock> = LazyLock::new(|| { @@ -924,9 +909,9 @@ mod tests { receiver.recv().await; if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { - assert_eq!(value.config_id, PATH_FIRST.config_id); - assert_eq!(value.source, PATH_FIRST.source); - assert_eq!(value.name, PATH_FIRST.name); + assert_eq!(value.path.config_id(), PATH_FIRST.config_id()); + assert_eq!(value.path.source(), PATH_FIRST.source()); + assert_eq!(value.path.name(), PATH_FIRST.name()); let parsed = value.data.as_ref().expect("dynamic config must parse"); if let Some(cfg) = parsed.downcast::() { assert!(matches!( @@ -943,10 +928,10 @@ mod tests { // just one update assert!(matches!(manager.fetch_update(), RemoteConfigUpdate::None)); - manager.unload_configs(&[PATH_FIRST.product]); + manager.unload_configs(&[PATH_FIRST.product()]); if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { - assert_eq!(value.config_id, PATH_FIRST.config_id); + assert_eq!(value.path.config_id(), PATH_FIRST.config_id()); } else { unreachable!(); } @@ -986,17 +971,17 @@ mod tests { // then the adds let was_second = if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { - value.config_id == PATH_SECOND.config_id + value.path.config_id() == PATH_SECOND.config_id() } else { unreachable!(); }; if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { assert_eq!( - &value.config_id, + value.path.config_id(), if was_second { - &PATH_FIRST.config_id + PATH_FIRST.config_id() } else { - &PATH_SECOND.config_id + PATH_SECOND.config_id() } ); } else { @@ -1020,11 +1005,11 @@ mod tests { // If we re-track it's added again immediately if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { assert_eq!( - &value.config_id, + value.path.config_id(), if was_second { - &PATH_SECOND.config_id + PATH_SECOND.config_id() } else { - &PATH_FIRST.config_id + PATH_FIRST.config_id() } ); } else { @@ -1113,9 +1098,9 @@ mod tests { receiver.recv().await; if let RemoteConfigUpdate::Add { value, .. } = manager.fetch_update() { - assert_eq!(value.config_id, PATH_LIVE_DEBUGGER.config_id); + assert_eq!(value.path.config_id(), PATH_LIVE_DEBUGGER.config_id()); assert_eq!( - value.product, + value.path.product(), RemoteConfigProduct::LiveDebugging, "must be parsed as LiveDebugger, not skipped" ); diff --git a/libdd-common/src/lib.rs b/libdd-common/src/lib.rs index 976de70584..2db7ced8ab 100644 --- a/libdd-common/src/lib.rs +++ b/libdd-common/src/lib.rs @@ -11,7 +11,7 @@ extern crate alloc; use alloc::borrow::Cow; use anyhow::Context; use core::{ops::Deref, str::FromStr}; -use http::uri; +use http::uri::{self, PathAndQuery, Uri}; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::path::PathBuf; @@ -351,6 +351,24 @@ impl Endpoint { /// Default value for the timeout field in milliseconds. pub const DEFAULT_TIMEOUT: u64 = 3_000; + pub fn agentless(site: &str, api_key: String) -> anyhow::Result { + Ok(Self { + url: Uri::builder() + .scheme("https") + .authority( + uri::Authority::try_from(site) + .with_context(|| format!("dd_site is an invalid url: {site}"))?, + ) + .path_and_query(PathAndQuery::from_static("")) + .build() + .with_context(|| format!("rc url is invalid for site: {site}"))?, + api_key: Some(api_key.into()), + timeout_ms: Self::DEFAULT_TIMEOUT, + test_token: None, + use_system_resolver: true, + }) + } + /// Returns an iterator of optional endpoint-specific headers (api-key, test-token) /// as (header_name, header_value) string tuples for any that are available. pub fn get_optional_headers(&self) -> impl Iterator { diff --git a/libdd-remote-config/Cargo.toml b/libdd-remote-config/Cargo.toml index b467c8a7cf..3b96291e45 100644 --- a/libdd-remote-config/Cargo.toml +++ b/libdd-remote-config/Cargo.toml @@ -11,21 +11,29 @@ description = "Datadog Remote Configuration client and config parsers" [features] default = ["client", "https"] +agentless = [ + "dep:tuf", + "dep:chrono", + "dep:prost", + "dep:rand", + "client", + "https", +] client = [ "libdd-trace-protobuf", - "libdd-capabilities", "bytes", "http-body-util", "http", "base64", "sha2", "uuid", + "dep:futures", "futures-util", "tokio", "tokio-util", "manual_future", "time", - "tracing" + "tracing", ] regex-lite = ["libdd-common/regex-lite"] @@ -39,10 +47,14 @@ test = ["hyper/server", "hyper-util"] anyhow.workspace = true bytes = { version = "1", optional = true } libdd-common = { path = "../libdd-common", version = "5.1.0", default-features = false } -libdd-capabilities = { path = "../libdd-capabilities", version = "2.1.0", optional = true } +libdd-capabilities = { path = "../libdd-capabilities", version = "2.1.0" } +libdd-capabilities-impl = { version = "3.0.0", path = "../libdd-capabilities-impl", default-features = false } libdd-trace-protobuf = { path = "../libdd-trace-protobuf", version = "4.0.0", optional = true } -hyper = { workspace = true, optional = true, features = ["http1", "client"] } -http-body-util = {version = "0.1", optional = true } +hyper = { workspace = true, optional = true, default-features = false, features = [ + "http1", + "client", +] } +http-body-util = { version = "0.1", optional = true } http = { version = "1.1", optional = true } base64 = { version = "0.22.1", optional = true } sha2 = { version = "0.10", optional = true } @@ -51,19 +63,36 @@ futures-util = { version = "0.3", optional = true } tokio = { workspace = true, optional = true, features = ["macros"] } tokio-util = { version = "0.7.10", optional = true } manual_future = { version = "0.1.1", optional = true } -time = { version = "0.3", features = ["parsing", "serde", "formatting"], optional = true } +time = { version = "0.3", features = [ + "parsing", + "serde", + "formatting", +], optional = true } tracing = { workspace = true, optional = true } -serde.workspace = true +serde = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } serde_with = "3" +rand = { version = "0.8.5", optional = true } thiserror = "2" hashbrown = "0.15" +# branch = "opw-develop" +tuf = { git = "https://github.com/DataDog/rust-tuf/", rev = "eb129ccad320b11e8bf99d2f0ff2c415a0795ccb", default-features = false, optional = true } +chrono = { version = "0.4", default-features = false, features = [ + "clock", +], optional = true } +prost = { version = "0.14.1", optional = true } +futures = { version = "0.3", features = ["executor"], optional = true } # `EnumIter` for external consumers to make struct available to runtime strum = { version = "0.26", default-features = false } strum_macros = "0.26" # Test feature -hyper-util = { workspace = true, features = ["http1", "client", "client-legacy", "service"], optional = true } +hyper-util = { workspace = true, features = [ + "http1", + "client", + "client-legacy", + "service", +], optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] libdd-capabilities-impl = { path = "../libdd-capabilities-impl", version = "3.0.0", default-features = false } diff --git a/libdd-remote-config/examples/remote_config_agentless_bench.rs b/libdd-remote-config/examples/remote_config_agentless_bench.rs new file mode 100644 index 0000000000..20dc9872e7 --- /dev/null +++ b/libdd-remote-config/examples/remote_config_agentless_bench.rs @@ -0,0 +1,306 @@ +// Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#![allow(unreachable_code)] +#![allow(unused)] + +//! Benchmark for agentless Remote Config fetching. +//! +//! Measures, for three distinct phases, three quantities each: +//! +//! 1. Client init: `SingleChangesFetcher::new`, which performs the TUF root bootstrap when +//! running in agentless mode. +//! 2. Initial fetch: the first call to `fetch_changes` on a freshly built client. +//! 3. Refetch: the second call to `fetch_changes`, with the client already warm. +//! +//! For each phase we report: +//! +//! * Wall-clock time: total elapsed time, end-to-end. +//! * Poll/CPU time: sum of time spent inside `Future::poll` calls on the current thread. This +//! is an approximation of the active computation time (parsing, TUF verification, request +//! building, response decoding, ...). +//! * Await/IO time: `wall - poll`, the time the future spent suspended waiting for IO (DNS, +//! TCP, TLS handshake, server response, ...). +//! +//! The instrumentation works by polling the benchmarked future manually on a `current_thread` +//! tokio runtime and accumulating the duration of each `poll()` invocation. No additional +//! dependencies are required. +//! +//! Usage: +//! DD_API_KEY=... DD_SITE=datadoghq.com \ +//! cargo run --release --example remote_config_agentless_bench \ +//! -p libdd-remote-config --features agentless +//! +//! Without `DD_API_KEY` / `DD_SITE`, this example exits — agentless mode is required. + +use libdd_capabilities::HttpClientCapability as _; +use libdd_common::Endpoint; +use libdd_remote_config::fetch::{ConfigInvariants, ConfigOptions, SingleChangesFetcher}; +use libdd_remote_config::file_storage::ParsedFileStorage; +use libdd_remote_config::RemoteConfigProduct::ApmTracing; +use libdd_remote_config::Target; +use std::future::Future; +use std::pin::Pin; +use std::process::Command; +use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +#[cfg(feature = "agentless")] +use libdd_remote_config::fetch::AgentlessConfig; + +const RUNTIME_ID: &str = "23e76587-5ae1-410c-a05c-137cae600a10"; +const SERVICE: &str = "bench-service"; +const ENV: &str = "bench-env"; +const VERSION: &str = "1.2.3"; + +fn get_hostname() -> String { + Command::new("hostname") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_else(|| "unknown".to_string()) +} + +/// A future wrapper that accumulates the time spent inside each `poll()` call into +/// `*poll_time`. The wall time is the elapsed between calling `Instrumented::new` (or just +/// before `.await`) and the future completing. +struct Instrumented<'a, F> { + inner: F, + poll_time: &'a mut Duration, +} + +impl<'a, F: Future + Unpin> Instrumented<'a, F> { + fn new(inner: F, poll_time: &'a mut Duration) -> Self { + *poll_time = Duration::ZERO; + Self { inner, poll_time } + } +} + +impl Future for Instrumented<'_, F> { + type Output = F::Output; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let start = Instant::now(); + let res = Pin::new(&mut self.inner).poll(cx); + let elapsed = start.elapsed(); + *self.poll_time += elapsed; + res + } +} + +#[derive(Default, Clone, Copy)] +struct Sample { + wall: Duration, + poll: Duration, +} + +impl Sample { + fn io(&self) -> Duration { + self.wall.saturating_sub(self.poll) + } +} + +fn print_row(label: &str, s: Sample) { + let wall_ms = s.wall.as_secs_f64() * 1000.0; + let poll_ms = s.poll.as_secs_f64() * 1000.0; + let io_ms = s.io().as_secs_f64() * 1000.0; + let poll_pct = if s.wall.as_nanos() > 0 { + 100.0 * s.poll.as_secs_f64() / s.wall.as_secs_f64() + } else { + 0.0 + }; + println!( + " {label:<18} wall = {wall_ms:>9.3} ms poll/CPU = {poll_ms:>9.3} ms \ + await/IO = {io_ms:>9.3} ms ({poll_pct:>5.1}% poll)" + ); +} + +async fn run_one_iteration( + iter: usize, + endpoint: Endpoint, + #[cfg(feature = "agentless")] agentless: Option, + #[cfg(not(feature = "agentless"))] agentless: Option, +) -> anyhow::Result<(Sample, Sample, Sample)> { + let target = Target::new( + SERVICE.to_string(), + ENV.to_string(), + VERSION.to_string(), + vec!["bench:true".to_string()], + vec![], + ); + + let options = ConfigOptions { + invariants: ConfigInvariants { + language: "benchlang".to_string(), + tracer_version: "0.0.1".to_string(), + endpoint, + agentless, + }, + products: vec![ApmTracing], + capabilities: vec![], + }; + + // --- 1. Client init: TUF root bootstrap in agentless mode --- + let mut init_poll = Duration::ZERO; + let init_wall_start = Instant::now(); + let fetcher_fut = Box::pin(SingleChangesFetcher::new( + ParsedFileStorage::default(), + target, + RUNTIME_ID.to_string(), + options, + libdd_capabilities_impl::NativeCapabilities::new_without_connection_pooling(), + )); + let mut fetcher = Instrumented::new(fetcher_fut, &mut init_poll).await?; + let init = Sample { + wall: init_wall_start.elapsed(), + poll: init_poll, + }; + + // --- 2. Initial fetch: first call to fetch_changes on a fresh client --- + let mut first_poll = Duration::ZERO; + let first_wall_start = Instant::now(); + // R is inferred from `ParsedFileStorage`'s `UpdatedFiles` impl. + let first_fut = Box::pin(fetcher.fetch_changes()); + let first_changes: Vec<_> = Instrumented::new(first_fut, &mut first_poll).await?; + let first = Sample { + wall: first_wall_start.elapsed(), + poll: first_poll, + }; + + // --- 3. Refetch: second call to fetch_changes (warm client) --- + let mut refetch_poll = Duration::ZERO; + let refetch_wall_start = Instant::now(); + let refetch_fut = Box::pin(fetcher.fetch_changes()); + let refetch_changes: Vec<_> = Instrumented::new(refetch_fut, &mut refetch_poll).await?; + let refetch = Sample { + wall: refetch_wall_start.elapsed(), + poll: refetch_poll, + }; + + println!( + "Iteration #{iter}: initial fetch returned {} change(s), refetch returned {} change(s)", + first_changes.len(), + refetch_changes.len(), + ); + + Ok((init, first, refetch)) +} + +#[cfg(feature = "agentless")] +async fn agentless_main() -> anyhow::Result<()> { + let hostname = get_hostname(); + println!("Hostname: {hostname}"); + + let dd_api_key = std::env::var("DD_API_KEY").ok(); + let dd_site = std::env::var("DD_SITE").ok(); + let iterations: usize = std::env::var("BENCH_ITERATIONS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5); + + let (endpoint, agentless): (Endpoint, Option<_>) = match (dd_api_key, dd_site) { + #[cfg(feature = "agentless")] + (Some(api_key), Some(site)) => { + println!("Agentless mode enabled (site: {site})"); + let endpoint = Endpoint::agentless(&site, api_key) + .expect("Failed to build agentless endpoint from DD_SITE"); + let agentless = AgentlessConfig::new(hostname.clone(), &endpoint) + .expect("Failed to build AgentlessConfig from DD_SITE"); + (endpoint, Some(agentless)) + } + #[cfg(not(feature = "agentless"))] + (Some(_), Some(_)) => { + eprintln!( + "This benchmark requires the `agentless` feature. \ + Re-run with: --features agentless" + ); + std::process::exit(1); + } + _ => { + eprintln!( + "DD_API_KEY and DD_SITE are required for the agentless benchmark.\n\ + Example:\n DD_API_KEY=... DD_SITE=datadoghq.com \\\n \ + cargo run --release --example remote_config_agentless_bench \\\n \ + -p libdd-remote-config --features agentless" + ); + std::process::exit(1); + } + }; + + println!("Running {iterations} iteration(s)\n"); + + let mut inits = Vec::with_capacity(iterations); + let mut firsts = Vec::with_capacity(iterations); + let mut refetches = Vec::with_capacity(iterations); + + for i in 0..iterations { + match run_one_iteration(i, endpoint.clone(), agentless.clone()).await { + Ok((init, first, refetch)) => { + print_row(" client init", init); + print_row(" initial fetch", first); + print_row(" refetch", refetch); + println!(); + inits.push(init); + firsts.push(first); + refetches.push(refetch); + } + Err(e) => { + eprintln!("Iteration {i} failed: {e:?}"); + } + } + } + + if inits.is_empty() { + anyhow::bail!("All iterations failed"); + } + + println!( + "=== Summary over {} successful iteration(s) ===", + inits.len() + ); + print_summary("client init", &inits); + print_summary("initial fetch", &firsts); + print_summary("refetch", &refetches); + + Ok(()) +} + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + #[cfg(feature = "agentless")] + agentless_main().await?; + Ok(()) +} + +fn print_summary(label: &str, samples: &[Sample]) { + let n = samples.len() as u32; + let sum_wall: Duration = samples.iter().map(|s| s.wall).sum(); + let sum_poll: Duration = samples.iter().map(|s| s.poll).sum(); + let avg = Sample { + wall: sum_wall / n, + poll: sum_poll / n, + }; + + let min_wall = samples.iter().map(|s| s.wall).min().unwrap(); + let max_wall = samples.iter().map(|s| s.wall).max().unwrap(); + let min_poll = samples.iter().map(|s| s.poll).min().unwrap(); + let max_poll = samples.iter().map(|s| s.poll).max().unwrap(); + + println!("{label}:"); + print_row("avg", avg); + print_row( + "min", + Sample { + wall: min_wall, + poll: min_poll, + }, + ); + print_row( + "max", + Sample { + wall: max_wall, + poll: max_poll, + }, + ); +} diff --git a/libdd-remote-config/examples/remote_config_fetch.rs b/libdd-remote-config/examples/remote_config_fetch.rs index d34474043a..a85b71b667 100644 --- a/libdd-remote-config/examples/remote_config_fetch.rs +++ b/libdd-remote-config/examples/remote_config_fetch.rs @@ -9,7 +9,7 @@ use libdd_remote_config::file_change_tracker::{Change, FilePath}; use libdd_remote_config::file_storage::ParsedFileStorage; use libdd_remote_config::RemoteConfigProduct::ApmTracing; use libdd_remote_config::{RemoteConfigParsed, Target}; -use std::time::Duration; +use std::process::Command; use tokio::time::sleep; const RUNTIME_ID: &str = "23e76587-5ae1-410c-a05c-137cae600a10"; @@ -17,8 +17,66 @@ const SERVICE: &str = "testservice"; const ENV: &str = "testenv"; const VERSION: &str = "1.2.3"; +fn get_hostname() -> String { + Command::new("hostname") + .output() + .ok() + .and_then(|o| String::from_utf8(o.stdout).ok()) + .map(|s| s.trim().to_string()) + .unwrap_or_else(|| "unknown".to_string()) +} + #[tokio::main(flavor = "current_thread")] async fn main() { + let hostname = get_hostname(); + println!("Hostname: {hostname}"); + + let dd_api_key = std::env::var("DD_API_KEY").ok(); + let dd_site = std::env::var("DD_SITE").ok(); + + let (endpoint, agentless) = match (dd_api_key, dd_site) { + (Some(api_key), Some(site)) => { + #[cfg(feature = "agentless")] + { + use libdd_remote_config::fetch::AgentlessConfig; + println!("DD_API_KEY and DD_SITE are set — enabling agentless mode (site: {site})"); + let endpoint = Endpoint::agentless(&site, api_key) + .expect("Failed to build agentless endpoint from DD_SITE"); + let agentless = AgentlessConfig::new(hostname, &endpoint) + .expect("Failed to build AgentlessConfig from DD_SITE"); + (endpoint, Some(agentless)) + } + #[cfg(not(feature = "agentless"))] + { + let _ = (api_key, site); + println!("DD_API_KEY and DD_SITE are set but agentless feature not enabled"); + ( + Endpoint { + url: http::Uri::from_static("http://localhost:8126"), + api_key: None, + timeout_ms: 5000, // custom timeout, defaults to 3 seconds + test_token: None, + ..Default::default() + }, + None, + ) + } + } + _ => { + println!("DD_API_KEY / DD_SITE not set — connecting to local agent"); + ( + Endpoint { + url: http::Uri::from_static("http://localhost:8126"), + api_key: None, + timeout_ms: 5000, // custom timeout, defaults to 3 seconds + test_token: None, + ..Default::default() + }, + None, + ) + } + }; + // SingleChangesFetcher is ideal for a single static (runtime_id, service, env, version) tuple // Otherwise a SharedFetcher (or even a MultiTargetFetcher for a potentially high number of // targets) for multiple targets is needed. These can be manually wired together with a @@ -41,19 +99,16 @@ async fn main() { invariants: ConfigInvariants { language: "awesomelang".to_string(), tracer_version: "99.10.5".to_string(), - endpoint: Endpoint { - url: http::Uri::from_static("http://localhost:8126"), - api_key: None, - timeout_ms: 5000, // custom timeout, defaults to 3 seconds - test_token: None, - ..Default::default() - }, + endpoint, + agentless, }, products: vec![ApmTracing], capabilities: vec![], }, NativeCapabilities::new_without_connection_pooling(), - ); + ) + .await + .expect("Failed to create SingleChangesFetcher"); loop { match fetcher.fetch_changes().await { @@ -84,7 +139,7 @@ async fn main() { } } - sleep(Duration::from_secs(1)).await; + sleep(fetcher.get_refresh_interval()).await; } } diff --git a/libdd-remote-config/roots/gov/config_root.json b/libdd-remote-config/roots/gov/config_root.json new file mode 100644 index 0000000000..839a3d50a5 --- /dev/null +++ b/libdd-remote-config/roots/gov/config_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 3, + "expires": "2026-10-31T17:00:00Z", + "keys": { + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAES2cvfa8r3HZ4AQeDUurdth7xqFk3\nqOuYtR877knUfOtJe+xU/F/ESVrl4B0ZMcyF3TaucgMsae4OVlc2lAW3Nw==\n-----END PUBLIC KEY-----\n" + } + }, + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETSYtYSgft/owrcf+DsvGzdl+wpSr\nAVe0hmZL/hvdC0oogI98nYTYzycP0B5M5xBeP4ZfJm/mlFFsqCHHosQWkA==\n-----END PUBLIC KEY-----\n" + } + } + }, + "roles": { + "root": { + "keyids": [ + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993", + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993", + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993", + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993", + "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "fe9b1451a0446f049888c4ece57fa4c8127f50cc2401d0bb15712e9367953425", + "sig": "304502202d297939cec401370527b9c7090942d3d3f7f38809e7e5d82f9283ca5f96f364022100b3f2e5f82d96e6034a602480e7470122e28901a8584a93563b0a2644bb9d9544" + }, + { + "keyid": "8907affe5835f969ee7680dda2e5b0ece95d839611d481483d89c22b2df42993", + "sig": "3045022100a4f10a6cc39a03d01328f15c5c4c9330edb295e5f5163929944ee9fa373aad56022025c6cccead7829f5c01d9811a9118e5fbecab325d298a506d96e5c705c193693" + } + ] +} diff --git a/libdd-remote-config/roots/gov/director_root.json b/libdd-remote-config/roots/gov/director_root.json new file mode 100644 index 0000000000..515e0f6a93 --- /dev/null +++ b/libdd-remote-config/roots/gov/director_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 3, + "expires": "2026-10-31T17:00:00Z", + "keys": { + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEUDdWZozMy6DojzrxkhevLhLzom0E\nnW0C7JWPXgnoL58OHhqDTHhkiUP5H3+fGdVKZ33Vca686aWWSwZUY6xSRQ==\n-----END PUBLIC KEY-----\n" + } + }, + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835": { + "keytype": "ecdsa", + "scheme": "ecdsa-sha2-nistp256", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "-----BEGIN PUBLIC KEY-----\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEE/Evc+4Qx1yPwe0SyvP52C9z8inY\ncTH0eCXHRu+mzShDx7Ne8gyA/vU696i9jcc4pfsOwo1WpIkJsXuqP0jG6A==\n-----END PUBLIC KEY-----\n" + } + } + }, + "roles": { + "root": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "1bd43b99872bee114b2b1c33cff7afbdb8ccfc799751aaa11c2336f3540d1c10", + "sig": "3045022100d31c2a8767706ebe22b9bc2592e18fe4067ac430cf77dc92bca781b120338db1022050fe0ebb8a47abe9b4cc25384f288a046e33f959dd46acc449fe8eb3581d776e" + }, + { + "keyid": "3360f9a30c063542b2d193fe01854ea3b7ae92c641812b97ce01180bf150c835", + "sig": "304502205d9d39c9b04ee047addd2d72a06d129fda1a81c23e5044d8009e054f6cf2e7fa0221009586e2a872bb68fc508b0667a59ed110276f2cf7840de968ca9a9b3643dcdf81" + } + ] +} \ No newline at end of file diff --git a/libdd-remote-config/roots/prod/config_root.json b/libdd-remote-config/roots/prod/config_root.json new file mode 100644 index 0000000000..35bf212fa3 --- /dev/null +++ b/libdd-remote-config/roots/prod/config_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 16, + "expires": "2026-10-31T17:00:00Z", + "keys": { + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "91d413c791907aae0be739d94a1e5e59c5d5ba65a8bbc1fb2153a5680f2d5958" + } + }, + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "9323800f89d833ee263d3661c2616da89e405b92beeec334f21d54b5f60fbd85" + } + } + }, + "roles": { + "root": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "620dacb7dc843acc731e4483c24ceb4121f4de5545f92d15dc2b13299b660e01", + "sig": "a8b4ee59576c82bc1bc944df014bbeb90f5cba4ffbd8b7878461da2c934fd3bf93ac4c3b85a7936584da4a5a0cfe93b7150b559fc96423a98a70a11fc844f208" + }, + { + "keyid": "e1fdd5827bb44defe9b87ed835c854be4b78a86ded013d1646bc416c1c89a9db", + "sig": "3332e240a023dc267e87e210c7b46b9fa5772932d84936e3a7a5b5018b0f45fbf068ce60b97beb6e7e6c0c12a68d68a44461e590a934b577c71d4ff6dd94db09" + } + ] +} diff --git a/libdd-remote-config/roots/prod/director_root.json b/libdd-remote-config/roots/prod/director_root.json new file mode 100644 index 0000000000..c011b9271f --- /dev/null +++ b/libdd-remote-config/roots/prod/director_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 15, + "expires": "2026-10-31T17:00:00Z", + "keys": { + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "286d6ae328365afec0f92519ceab68cd627e34072cde90b2f5d167badea970f2" + } + }, + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "afdd68be53815d67f8fa99cf101aac4589a358c660adf7dd4e179fe96834d3c9" + } + } + }, + "roles": { + "root": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "b2b93a6dccc96d053e6db39181124c85ba4156d43503d4351b5500316fa084e8", + "sig": "ccbe8cdd7dfb9a9d6b4bef8075a7aaf9baafe69a07100f22c04677a9737a23b24055ac3a0776c7021ae6a2fd175a251c0604164ea6705a0a896844766d2ecd07" + }, + { + "keyid": "44d70fa8eae4c07f26c2767270827b6b9e11e7972926b3b419b5ea14ec32f796", + "sig": "068a2e37e93688702e75ebb328b74cd8879832a63179ba1c54976aae4ee03a5e936c7b7274d4a6aa6755c27cfe800097984d94c83be901bde72103dccebcc008" + } + ] +} diff --git a/libdd-remote-config/roots/staging/config_root.json b/libdd-remote-config/roots/staging/config_root.json new file mode 100644 index 0000000000..233580c7f9 --- /dev/null +++ b/libdd-remote-config/roots/staging/config_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 31, + "expires": "2027-04-14T17:00:00Z", + "keys": { + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "09402247ef6252018e52c7ba6a3a484936f14dad6ae921c556a1d092f4a68f0f" + } + }, + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "cf248bc222a5dfc9676a2a3ef90526c84adb09649db56686705f69f42908d7d8" + } + } + }, + "roles": { + "root": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "6aac6a51efedb4e54915bf9fbd2cfb49fbf428d46052bcaf3c72409c33ecdf5e", + "sig": "ab1c738f3dbfa82f262906818743626e7ccba37cdfd785f237ee094da11f23b03aeb5d11e567e1ae1b557cd3585736f0ad30a11503b6f35fb461792f698ad60e" + }, + { + "keyid": "bd3ea764afdf757f07bab1e9e501a5fda1d49a8da3eaddc53a50dbe2aff92545", + "sig": "321896985525b004e0158c6a9775f89340953493bb2726a5d0160edcff3d0d162c3094b94fcc9fa4dda57cce7cbe5daff72e2aa01b77f5180627571f253ffd03" + } + ] +} diff --git a/libdd-remote-config/roots/staging/director_root.json b/libdd-remote-config/roots/staging/director_root.json new file mode 100644 index 0000000000..69517704bf --- /dev/null +++ b/libdd-remote-config/roots/staging/director_root.json @@ -0,0 +1,73 @@ +{ + "signed": { + "_type": "root", + "spec_version": "1.0", + "version": 27, + "expires": "2027-04-14T17:00:00Z", + "keys": { + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "f7c278f32e69ce7d5ca5b81bd2cbe2b4b44177eee36ed025ec06bd19e47eaefe" + } + }, + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid_hash_algorithms": [ + "sha256", + "sha512" + ], + "keyval": { + "public": "47be15ec10499208aa5ef9a1e32010cc05c047a98d18ad084d6e4e51baa1b93c" + } + } + }, + "roles": { + "root": { + "keyids": [ + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6", + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1" + ], + "threshold": 2 + }, + "snapshot": { + "keyids": [ + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6", + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1" + ], + "threshold": 2 + }, + "targets": { + "keyids": [ + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6", + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1" + ], + "threshold": 2 + }, + "timestamp": { + "keyids": [ + "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6", + "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1" + ], + "threshold": 2 + } + }, + "consistent_snapshot": true + }, + "signatures": [ + { + "keyid": "233a529fe7c63b5b9081f6e0e2681cc227f85e04ad434d0a165a2f69b87255a6", + "sig": "807f8757661fef7d036709585618ca5eed446fdabecda2bbc10c131411c9a96e8e1d9a08d80b804698083831409277405ba69410482c138646c9a59efe237b07" + }, + { + "keyid": "6ca796e7b4883af3bb3d522dc0009984dcbf5ad2a6c9ea354d30acc32d8b75d1", + "sig": "0a28ed83e9f9ec5403ea99187456b844c0c5bff62fc49b3ec833902b56b7eea22c1e74e40c70b988909d6fd732e846a2c830d83380d200cb5c9d65d675253602" + } + ] +} diff --git a/libdd-remote-config/src/fetch/agentless.rs b/libdd-remote-config/src/fetch/agentless.rs new file mode 100644 index 0000000000..294475c470 --- /dev/null +++ b/libdd-remote-config/src/fetch/agentless.rs @@ -0,0 +1,1657 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::fetch::FileStorage; +use crate::RemoteConfigPath; + +use std::{fmt, ops::RangeInclusive, time::Duration}; + +use anyhow::{bail, format_err}; +use base64::Engine; +use hashbrown::{HashMap, HashSet}; +use http::{ + header, + uri::{Authority, PathAndQuery}, + Method, Request, Uri, +}; +use libdd_capabilities::{Bytes, HttpClientCapability}; +use libdd_common::Endpoint; +use libdd_trace_protobuf::remoteconfig; +use prost::Message; +use serde_json::Value; +use tracing::debug; +use tuf::metadata::{ + Metadata, MetadataPath, MetadataVersion, RawSignedMetadata, TargetDescription, TargetPath, +}; +use tuf::repository::RepositoryStorage; + +/// Embedded TUF trust roots, per site +/// +/// These roots don't have to be unexpired to be used. On the first request made to the RC backend +/// we will fetch the trust chain starting from the embedded root. +/// +/// The trust chain will then be checked (root N signs root (N + 1)). The embedded root is used to +/// bootstrap the chain and only the latest, highest version root in the chain +/// needs to be unexpired. +mod roots { + pub const PROD_CONFIG: &[u8] = include_bytes!("../../roots/prod/config_root.json"); + + pub const PROD_DIRECTOR: &[u8] = include_bytes!("../../roots/prod/director_root.json"); + + pub const STAGING_CONFIG: &[u8] = include_bytes!("../../roots/staging/config_root.json"); + + pub const STAGING_DIRECTOR: &[u8] = include_bytes!("../../roots/staging/director_root.json"); + + pub const GOV_CONFIG: &[u8] = include_bytes!("../../roots/gov/config_root.json"); + + pub const GOV_DIRECTOR: &[u8] = include_bytes!("../../roots/gov/director_root.json"); +} + +/// Datadog site selection used to pick a default TUF trust-root pair. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Site { + Prod, + Staging, + Gov, +} + +impl Site { + /// Map an endpoint authority/host to a Datadog site. + /// + /// The configured agentless endpoint authority looks like `config.` + /// (see `make_agentless_configs_endpoint`), so we strip a leading + /// `config.` prefix and apply the same rules the agent uses. + fn from_host(host: &str) -> Self { + let site = host.strip_prefix("config.").unwrap_or(host); + if site == "datad0g.com" || site.ends_with(".datad0g.com") { + Site::Staging + } else if site == "ddog-gov.com" || site.ends_with(".ddog-gov.com") { + Site::Gov + } else { + Site::Prod + } + } + + fn embedded_config_root(self) -> &'static [u8] { + match self { + Site::Prod => roots::PROD_CONFIG, + Site::Staging => roots::STAGING_CONFIG, + Site::Gov => roots::GOV_CONFIG, + } + } + + fn embedded_director_root(self) -> &'static [u8] { + match self { + Site::Prod => roots::PROD_DIRECTOR, + Site::Staging => roots::STAGING_DIRECTOR, + Site::Gov => roots::GOV_DIRECTOR, + } + } +} + +/// Fake version sent to RC. We have to do this as the RC backend will not answer if the +/// agent_version field is empty or lower than a certain version. +/// +/// This is currently set to the last agent version released +const FAKE_AGENT_VERSION: &str = "7.1000.0-agentless"; + +type TUFRepo = tuf::repository::EphemeralRepository; +type TUFClient = tuf::client::Client; + +// Make a remote config API endpoint from and endpoint where `e.url` is the base dd site +// If the endpoint is not suitable (api key not set, not https), returns N +pub fn make_agentless_configs_endpoint(e: &Endpoint) -> Option { + let e = e.clone(); + if !(e.url.scheme_str().is_some_and(|s| s == "https") + && e.url.authority().is_some() + && e.api_key.is_some()) + { + return None; + } + + let mut parts = e.url.into_parts(); + parts.authority = + Some(Authority::try_from(format!("config.{}", parts.authority?.as_str())).ok()?); + parts.path_and_query = Some(PathAndQuery::from_static("/api/v0.1/configurations")); + + Some(Endpoint { + url: Uri::from_parts(parts).ok()?, + ..e + }) +} + +/// Configuration for agentless remote-config fetching. +/// +/// Construction is fallible: an `AgentlessConfig` value is proof that the +/// hostname is non-empty and that the caller-supplied endpoint has been +/// successfully rewritten to the agentless URL (`config./api/v0.1/ +/// configurations`). +/// +/// Downstream code (`ConfigFetcherState::with_client`, +/// `MultiTargetFetcher::new`, ...) can therefore consume this type +/// infallibly. +#[derive(Clone, Debug, Hash, Eq, PartialEq)] +pub struct AgentlessConfig { + /// Hostname reported to the RC backend. Guaranteed non-empty by [`Self::new`]. + hostname: String, + /// Pre-rewritten agentless endpoint (`config./api/v0.1/configurations`). + /// Guaranteed valid (https, authority present, API key present) by [`Self::new`]. + agentless_endpoint: Endpoint, + /// Override the `agent_uuid` field sent to the RC backend. + agent_uuid: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum AgentlessConfigError { + #[error("agentless enabled but hostname is empty")] + EmptyHostname, + #[error( + "agentless endpoint is invalid (missing https, authority or API key, \ + or the URI cannot be rewritten)" + )] + InvalidEndpoint, +} + +impl AgentlessConfig { + /// Returns [`AgentlessConfigError::EmptyHostname`] if `hostname` is empty + /// and [`AgentlessConfigError::InvalidEndpoint`] if `endpoint` does not + /// meet the agentless preconditions. + pub fn new(hostname: String, endpoint: &Endpoint) -> Result { + if hostname.is_empty() { + return Err(AgentlessConfigError::EmptyHostname); + } + let agentless_endpoint = make_agentless_configs_endpoint(endpoint) + .ok_or(AgentlessConfigError::InvalidEndpoint)?; + Ok(Self { + hostname, + agentless_endpoint, + agent_uuid: None, + }) + } + + /// Override the `agent_uuid` field sent to the RC backend. + #[must_use] + pub fn with_agent_uuid(mut self, agent_uuid: String) -> Self { + self.agent_uuid = Some(agent_uuid); + self + } + + pub fn hostname(&self) -> &str { + &self.hostname + } + + /// Pre-validated agentless endpoint. + pub fn agentless_endpoint(&self) -> &Endpoint { + &self.agentless_endpoint + } + + pub fn agent_uuid(&self) -> Option<&str> { + self.agent_uuid.as_deref() + } +} + +pub type NativeAgentlessFetcher = AgentlessFetcher; + +pub struct AgentlessFetcher { + http: C, + opaque_backend_state: Vec, + director_client: TUFClient, + config_client: TUFClient, + /// Last non-empty config top-targets metadata received from the backend. The + /// backend only re-sends config top-targets when their version changes; on a + /// config root rotation rust-tuf purges its trusted top-targets and must + /// re-fetch them from the remote repo, so we cache and re-serve the last copy + /// to avoid being stuck (incident-45734). + last_config_top_targets: Option, + /// Org UUID pinned to a config root version. Root rotation forces a + /// re-fetch, so a bad pin clears itself on the next rotation. + org_uuid: Option, + /// Whether the one-shot org-UUID prefetch has run. Cleared by `reset()`. + org_data_prefetched: bool, + hostname: String, + agent_uuid_override: Option, + products: HashSet, + refresh_interval: Duration, + /// Number of consecutive `fetch_config` failures. Reset to 0 on success. + consecutive_failures: u32, + endpoint: Endpoint, +} + +#[derive(Debug, Clone)] +struct OrgUuidBinding { + config_root_version: u64, + uuid: String, +} + +#[derive(Debug)] +pub struct ClientTargetRef { + pub path: RemoteConfigPath, + pub version: u64, + pub primary_hash: String, + pub length: u64, +} + +pub struct ClientResponse { + pub root_version: u64, + pub target_version: u64, + pub opaque_backend_state: Vec, + /// All currently active targets; content is already stored in the outer cache. + pub targets: Vec, + pub refresh_interval: Duration, +} + +/// A single TUF target that has passed metadata verification: it comes from the +/// director's trusted `targets` database (so its signature, snapshot binding and +/// version have already been checked by rust-tuf's `update()`), it is not +/// expired, and it declares at least one hash algorithm we can verify. +/// +/// This is a lightweight, already-validated *view* over a +/// [`tuf::metadata::TargetDescription`] — it borrows the target `path` and copies +/// out only the fields we need downstream. It is produced by [`trusted_targets`] +/// via [`TrustedTarget::try_create`]; construction fails (dropping the target) +/// if the target is expired or has no supported hash. +/// +/// Note: holding a `TrustedTarget` means the *metadata* is trusted, not that the +/// *content* has been fetched or checked — [`AgentlessFetcher::fetch_target`] +/// still re-verifies the downloaded bytes against `length` and `all_hashes`. +struct TrustedTarget<'a> { + path: &'a tuf::metadata::TargetPath, + parsed_path: RemoteConfigPath, + length: u64, + version: u64, + all_hashes: Vec<(&'static tuf::crypto::HashAlgorithm, tuf::crypto::HashValue)>, + /// Lowercase hex of the first supported hash; used for cache-hit comparisons. + primary_hash: String, +} + +const CUSTOM_METADATA_EXPIRY_PATH: &str = "expires"; + +impl<'a> TrustedTarget<'a> { + fn try_create(path: &'a TargetPath, desc: &'a TargetDescription) -> anyhow::Result { + let parsed_path = RemoteConfigPath::parse(path.as_str())?; + + if let Some(expiry) = desc.custom().get(CUSTOM_METADATA_EXPIRY_PATH) { + let expiry_ts = expiry + .as_u64() + .ok_or_else(|| format_err!("expiry not a number"))?; + + // Use saturating arithmetic so a far-future `expires` cannot overflow + // `u64` (which panics in debug builds and wraps to a fail-open value in + // release builds). Saturating to `u64::MAX` keeps genuinely far-future + // targets "not yet expired" while never wrapping below `now`. + if expiry_ts.saturating_mul(1000) <= now_unix_milli_ts() { + bail!("expired target at path: {path}") + } + } + + let all_hashes = tuf::crypto::retain_supported_hashes(desc.hashes()); + if all_hashes.is_empty() { + bail!("no supported hash algorithm for target at path: {path}") + } + // retain_supported_hashes return order is deterministic. + let primary_hash = all_hashes[0].1.to_string(); + + let version = desc.custom().get("v").and_then(|v| v.as_u64()).unwrap_or(0); + + Ok(Self { + path, + parsed_path, + length: desc.length(), + version, + all_hashes, + primary_hash, + }) + } +} + +impl AgentlessFetcher { + /// Create a new `AgentlessFetcher` client. + /// + /// # Errors + /// Returns an error if TUF root initialization fails. + pub async fn new( + cfg: AgentlessConfig, + endpoint: Endpoint, + http_client: C, + ) -> anyhow::Result { + // Pick the embedded trust roots based on the endpoint's host. + let site = endpoint + .url + .host() + .map(Site::from_host) + .unwrap_or(Site::Prod); + + Ok(Self { + endpoint, + http: http_client, + director_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(site.embedded_director_root().to_vec()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await?, + config_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(site.embedded_config_root().to_vec()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await?, + last_config_top_targets: None, + org_uuid: None, + org_data_prefetched: false, + hostname: cfg.hostname, + agent_uuid_override: cfg.agent_uuid, + products: HashSet::new(), + + opaque_backend_state: Vec::new(), + refresh_interval: Duration::from_secs(60), + consecutive_failures: 0, + }) + } + + /// Number of consecutive failed `fetch_config` calls. `0` after a success. + pub fn consecutive_failures(&self) -> u32 { + self.consecutive_failures + } + + /// Recommended delay before the next `fetch_config` attempt given the + /// current consecutive-failure count. Returns `None` when no backoff + /// applies (i.e. either no failures yet, or only a single one). + pub fn next_backoff(&self) -> Option { + compute_backoff(self.consecutive_failures) + } + + /// Drop all trusted non-root TUF metadata and per-fetch caches so a partially + /// advanced state from a mid-`apply()` failure cannot block subsequent polls. + /// + /// The trusted root of each client is preserved — including any version + /// reached via `Database::update_root` chaining past the embedded/override + /// root — so recovery does not force the backend to re-send the full root + /// rotation chain on the next poll. See `tuf::database::Database::purge_metadata` + /// The local and remote TUF repositories are also swapped for fresh empty ones so no stale + /// pre-failure metadata can be picked up by the next `update()`. + fn reset(&mut self) { + self.director_client.purge_metadata(); + *self.director_client.local_repo_mut() = TUFRepo::new(); + *self.director_client.remote_repo_mut() = TUFRepo::new(); + self.config_client.purge_metadata(); + *self.config_client.local_repo_mut() = TUFRepo::new(); + *self.config_client.remote_repo_mut() = TUFRepo::new(); + self.products.clear(); + self.opaque_backend_state.clear(); + self.last_config_top_targets = None; + self.org_uuid = None; + self.org_data_prefetched = false; + } + + /// Check the config snapshot's `custom.org_uuid` against the UUID served + /// by `/api/v0.1/org`. No snapshot custom => skip. + /// + /// The pinned UUID is keyed by the config trusted-root version, so a root + /// rotation forces a fresh fetch. `prefetched` reuses the first-poll + /// concurrent fetch; otherwise the UUID is fetched here. + async fn verify_org_uuid(&mut self, prefetched: Option) -> anyhow::Result<()> { + let Some(expected) = self + .config_client + .database() + .trusted_snapshot() + .ok_or_else(|| format_err!("org UUID check failed: missing trusted snapshot"))? + .additional_fields() + .get("custom") + .and_then(|c| c.get("org_uuid")) + .and_then(Value::as_str) + else { + return Ok(()); + }; + + let root_version = self.config_client.database().trusted_root().version(); + + let stored: &str = match self.org_uuid.as_ref() { + Some(b) if b.config_root_version == root_version => &b.uuid, + _ => { + let uuid = match prefetched { + Some(u) => u, + None => self.get_org_data().await?.uuid, + }; + &self + .org_uuid + .insert(OrgUuidBinding { + config_root_version: root_version, + uuid, + }) + .uuid + } + }; + + anyhow::ensure!( + stored == expected, + "org UUID mismatch: intake={stored} snapshot={expected}" + ); + Ok(()) + } + + fn fetch_target<'a>(&'a self, target: &TrustedTarget<'_>) -> anyhow::Result<&'a [u8]> { + let target_path = target.path; + + // Fetch from the remote __unverified__ repo. + // This is fine because we compare the hash+len against TUF-validated metadata. + let buf = self.director_client.remote_repo().get_target(target_path)?; + + let actual_len = u64::try_from(buf.len()).map_err(|_| { + let got = buf.len(); + format_err!("target length overflows u64 for path: {target_path} (got {got} bytes)") + })?; + if actual_len != target.length { + bail!("bad length for file at path: {target_path}") + } + + let hash_algs = target + .all_hashes + .iter() + .map(|(alg, _val)| (*alg).clone()) + .collect::>(); + let actual_hashes = tuf::crypto::calculate_hashes_from_slice(buf, hash_algs.as_slice())?; + let expected: HashMap<_, _> = target + .all_hashes + .iter() + .map(|(alg, val)| (alg, val)) + .collect(); + + if !(actual_hashes.len() == expected.len() + && actual_hashes + .iter() + .all(|(k, v)| expected.get(&k).is_some_and(|e| *e == v))) + { + bail!("hash did not match: {target_path}") + } + + Ok(buf) + } + + /// Fetch remote config. Newly-downloaded target content is written into + /// `cache` after having been validated. + /// The [`ClientResponse`] contains the path, and metadata info of targets + /// that have been sent to the client. + /// + /// This separation between content and metadata is done for 2 reasons: + /// 1. Remote Config does not re-send target files that we have received in a previous fetch. So + /// it should already be in cache. + /// 2. Currently we only have a single [`remoteconfig::Client`], but targets for mutliple + /// clients can be fetched at once, and we need to do a M:N mapping of target files to + /// clients, with the same target that can be used by mutliple client + pub(crate) async fn fetch_config( + &mut self, + c: remoteconfig::Client, + cache: &TargetCache<'_, Storage>, + ) -> anyhow::Result { + // Derive the versions we report to the backend directly from the live + // trusted databases. A freshly built or just-reset client has no + // trusted snapshot yet, so it reports snapshot version 0 and the embedded + // root versions. + let current_config_snapshot_version = self + .config_client + .database() + .trusted_snapshot() + .map_or(0, |s| s.version()); + let current_config_root_version = self.config_client.database().trusted_root().version(); + let current_director_root_version = + self.director_client.database().trusted_root().version(); + + let mut all_products: HashSet = HashSet::new(); + for p in &c.products { + all_products.get_or_insert_with(p, String::clone); + } + let new_products = all_products + .difference(&self.products) + .cloned() + .collect::>(); + let old_products = self + .products + .intersection(&all_products) + .cloned() + .collect::>(); + + let now = now_unix_milli_ts(); + + let (has_error, error) = match c.state.as_ref() { + Some(state) if state.has_error => (true, state.error.clone()), + _ => (false, String::new()), + }; + + let request = remoteconfig::LatestConfigsRequest { + hostname: self.hostname.clone(), + current_config_snapshot_version, + current_config_root_version, + current_director_root_version, + products: old_products, + new_products, + backend_client_state: self.opaque_backend_state.clone(), + active_clients: vec![remoteconfig::Client { + last_seen: now, + ..c + }], + agent_version: FAKE_AGENT_VERSION.to_owned(), + has_error, + error, + trace_agent_env: String::new(), + org_uuid: String::new(), + tags: vec![], + agent_uuid: self + .agent_uuid_override + .as_deref() + .unwrap_or_else(|| libdd_common::machine_id::get_machine_id()) + .to_owned(), + }; + // During first poll only fetch org data in parallel with the config request + // to hide its latency. Later polls (and prefetch failures) fall back to + // the sequential fetch in `verify_org_uuid`. + let (response_result, prefetched_org_uuid) = if !self.org_data_prefetched { + self.org_data_prefetched = true; + let (r, org) = futures::join!(self.get_latest_config(request), self.get_org_data()); + let org_uuid = match org { + Ok(d) => Some(d.uuid), + Err(e) => { + debug!("org data prefetch failed, will fetch lazily: {e:#}"); + None + } + }; + (r, org_uuid) + } else { + (self.get_latest_config(request).await, None) + }; + let response = match response_result { + Ok(r) => r, + Err(e) => { + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + return Err(e); + } + }; + + let active_targets = match self.apply(&response, cache, prefetched_org_uuid).await { + Ok(t) => t, + Err(e) => { + self.consecutive_failures = self.consecutive_failures.saturating_add(1); + // On any `apply()` failure the trusted databases may have been advanced + // in place and incrementally, leaving them inconsistent with the + // versions we would report next poll. + // Reset both clients so the next poll restarts from a clean state + self.reset(); + return Err(e); + } + }; + self.consecutive_failures = 0; + + self.products = all_products; + + // TODO: + // In the future we will want to query configs for multiple clients (for PHP, which can have + // many processes use the same rc client). + // This means we will need to dispatch the different files based on filter predicates + // which we currently do not parse. + + Ok(ClientResponse { + root_version: self.config_client.database().trusted_root().version(), + target_version: self + .config_client + .database() + .trusted_targets() + .ok_or(anyhow::anyhow!("Missing target data"))? + .version(), + opaque_backend_state: self.opaque_backend_state.clone(), + targets: active_targets, + refresh_interval: self.refresh_interval, + }) + } + + async fn get_org_data(&self) -> anyhow::Result { + let path = PathAndQuery::from_static("/api/v0.1/org"); + let res = self.send_request(Method::GET, path, Bytes::new()).await?; + parse_rc_response(res) + } + + /// Fetch the latest Remote Config for this client. + /// + /// # Errors + /// Returns an error if the HTTP request fails or the response cannot be decoded. + async fn get_latest_config( + &self, + req: remoteconfig::LatestConfigsRequest, + ) -> anyhow::Result { + let path = PathAndQuery::from_static("/api/v0.1/configurations"); + let body = Bytes::from(req.encode_to_vec()); + let res = self.send_request(Method::POST, path, body).await?; + let res = parse_rc_response(res)?; + Ok(res) + } + + #[allow(clippy::future_not_send)] + async fn send_request( + &self, + method: Method, + path: PathAndQuery, + body: Bytes, + ) -> anyhow::Result> { + let req = self + .endpoint + .set_standard_headers( + Request::builder(), + concat!("Libdatadog/", env!("CARGO_PKG_VERSION")), + ) + .header(header::CONTENT_TYPE, "application/x-protobuf") + .uri(url_with_path(self.endpoint.url.clone(), path)?) + .method(method) + .body(body)?; + let timeout = Duration::from_millis(self.endpoint.timeout_ms); + let response = tokio::time::timeout(timeout, self.http.request(req)) + .await + .map_err(|_| { + format_err!( + "Remote config request timed out after {}ms", + self.endpoint.timeout_ms, + ) + })??; + Ok(response) + } + + /// Update the TUF-clients state to add the new data fetched from the intake, + /// verify it with tuf-rust and verify target files against the TUF signed information. + /// + /// After this function returns Ok, the TUF client trusted database should be in-sync + /// with data fetched from the backend + async fn apply( + &mut self, + response: &remoteconfig::LatestConfigsResponse, + cache: &TargetCache<'_, S>, + prefetched_org_uuid: Option, + ) -> anyhow::Result> { + // At a high level, we're populating the "remote" repos with the metadata + // that we received from upstream (which does not validate it), and then using the clients' + // `update` methods to synchronize that metadata to the "local" repos, during which + // validation is performed. + + let root_path = MetadataPath::root(); + let timestamp_path = MetadataPath::timestamp(); + let snapshot_path = MetadataPath::snapshot(); + let targets_path = MetadataPath::targets(); + + let Some(metas) = response.config_metas.as_ref() else { + bail!("missing config meta from LatestConfigsResponse") + }; + // The backend diffs config top-targets against the snapshot version we + // report: it sends them only when their version changed, otherwise the + // field is empty. + // + // rust-tuf purges its trusted top-targets whenever the + // config root rotates and then re-fetches them from the remote repo; if + // we wipe the remote repo (below) and the backend sent none, that + // re-fetch finds nothing and `update()` is stuck (incident-45734). Re-serve + // the last cached copy when the response omits them. + let config_top_targets = if metas.top_targets.is_some() { + &metas.top_targets + } else { + &self.last_config_top_targets + }; + + let config_repo_mut = self.config_client.remote_repo_mut(); + *config_repo_mut = TUFRepo::new(); + + store(config_repo_mut, &root_path, &metas.roots).await?; + store_noversion(config_repo_mut, ×tamp_path, &metas.timestamp).await?; + store(config_repo_mut, &snapshot_path, &metas.snapshot).await?; + store(config_repo_mut, &targets_path, config_top_targets).await?; + // Delegated targets are stored later, after verifying the top-level signatures. + + let Some(metas) = response.director_metas.as_ref() else { + bail!("missing director meta from LatestConfigsResponse") + }; + + let director_remote_repo = self.director_client.remote_repo_mut(); + *director_remote_repo = TUFRepo::new(); + for target_file in &response.target_files { + let trimmed_path = trim_hash_target_path(&target_file.path)?; + director_remote_repo + .store_target( + &TargetPath::new(&trimmed_path)?, + &mut target_file.raw.as_slice(), + ) + .await?; + } + + store(director_remote_repo, &root_path, &metas.roots).await?; + store_noversion(director_remote_repo, ×tamp_path, &metas.timestamp).await?; + store(director_remote_repo, &snapshot_path, &metas.snapshot).await?; + store(director_remote_repo, &targets_path, &metas.targets).await?; + + // Verification of top level metadata for each individual repo happens here + self.config_client.update().await?; + self.director_client.update().await?; + + let now = chrono::Utc::now(); + let parent = MetadataPath::targets(); + + // Ingest each delegated targets blob into the config DB. This enforces the + // per-product signing keys: `update_delegated_targets` verifies signatures, + // expiry and version monotonicity before inserting into `trusted_delegations`. + if let Some(metas) = response.config_metas.as_ref() { + for dm in &metas.delegated_targets { + let role = MetadataPath::new(dm.role.clone()) + .map_err(|e| format_err!("bad delegated role name {:?}: {e}", dm.role))?; + let raw = RawSignedMetadata::new(dm.raw.clone()); + self.config_client + .database_mut() + .update_delegated_targets(&now, &parent, &role, &raw) + .map_err(|e: tuf::Error| { + format_err!("failed to verify config delegation {}: {e}", dm.role) + })?; + } + } + + // Enforce that each director-announced target is also authorized by the + // config repo's per-product delegated keys, not just the single director key. + verify_director_against_config(&self.config_client, &self.director_client)?; + self.verify_org_uuid(prefetched_org_uuid).await?; + + let targets: Vec> = trusted_targets(&self.director_client)?.collect(); + + // Hold the cache lock for the entire fetch/verify/store cycle. + // Prevents another agentless fetcher sharing the same + // `ConfigFetcherState` from interleaving between `is_cached_batch` + // and `store_batch` + let mut files_guard = cache.lock(); + + let cached_paths: HashSet<&RemoteConfigPath> = files_guard.is_cached_batch( + targets + .iter() + .map(|t| (&t.parsed_path, t.primary_hash.as_str(), t.length)), + ); + + let mut new_targets: Vec = Vec::new(); + for t in &targets { + if cached_paths.contains(&t.parsed_path) { + continue; + } + let content = self.fetch_target(t)?; + new_targets.push(NewTarget { + path: t.parsed_path.clone(), + version: t.version, + primary_hash: t.primary_hash.clone(), + hashes: t + .all_hashes + .iter() + .map(|(alg, hash)| (hash_algorithm_to_str(alg).to_owned(), hash.to_string())) + .collect(), + content: content.to_owned(), + }); + } + files_guard.store_batch(new_targets)?; + + let active_paths: HashSet<&RemoteConfigPath> = + targets.iter().map(|t| &t.parsed_path).collect(); + files_guard.retain_only(&active_paths); + + let active_targets: Vec = targets + .iter() + .map(|t| ClientTargetRef { + path: t.parsed_path.clone(), + version: t.version, + primary_hash: t.primary_hash.clone(), + length: t.length, + }) + .collect(); + + // The Remote Config service uses a `custom` field at the top-level of the targets + // metadata to store this field which we are supposed to echo back to the server. + if let Some((opaque_backend_state, refresh_interval)) = + get_director_custom(&self.director_client) + { + if let Some(opaque_backend_state) = opaque_backend_state { + self.opaque_backend_state = opaque_backend_state; + } + if let Some(refresh_interval) = refresh_interval { + self.refresh_interval = refresh_interval; + } + } + + // Commit the top-targets cache only now that `apply()` has fully + // succeeded, so a mid-way failure (followed by `reset()`) never leaves a + // stale cached copy + if let Some(config_metas) = response.config_metas.as_ref() { + if config_metas.top_targets.is_some() { + self.last_config_top_targets = config_metas.top_targets.clone(); + } + } + + Ok(active_targets) + } +} + +const REFRESH_INTERVAL_BOUNDS: RangeInclusive = + Duration::from_secs(1)..=Duration::from_secs(60); + +fn get_director_custom(director_client: &TUFClient) -> Option<(Option>, Option)> { + let custom = director_client + .database() + .trusted_targets()? + .additional_fields() + .get("custom")?; + + Some(( + custom + .get("opaque_backend_state") + .and_then(Value::as_str) + .and_then(|s| base64::engine::general_purpose::STANDARD.decode(s).ok()), + custom + .get("agent_refresh_interval") + .and_then(Value::as_u64) + .map(Duration::from_secs) + // Mirror the agent: silently drop values outside `[1s, 1m]` + .filter(|d| REFRESH_INTERVAL_BOUNDS.contains(d)), + )) +} + +fn url_with_path(base: http::Uri, path: PathAndQuery) -> anyhow::Result { + let mut parts = base.into_parts(); + parts.path_and_query = Some(path); + Ok(http::Uri::from_parts(parts)?) +} + +fn parse_rc_response( + response: http::Response, +) -> anyhow::Result { + let status = response.status().as_u16(); + let body = response.into_body(); + if !(200..300).contains(&status) { + let body_lossy = String::from_utf8_lossy(&body); + bail!("Non 2XX status code: {status}\n{body_lossy}") + } + Ok(T::decode(body)?) +} + +/// Compute the backoff delay to wait before the next `fetch_config` attempt, +/// given the number of consecutive failures observed so far. +fn compute_backoff(consecutive_failures: u32) -> Option { + (consecutive_failures != 0).then(|| { + let consecutive_failures = std::cmp::min(3, consecutive_failures) as u64; + let base = 30 * consecutive_failures; + jitter_secs(base, base + 30) + }) +} + +/// Random duration in `[min_secs, max_secs]` +fn jitter_secs(min_secs: u64, max_secs: u64) -> Duration { + use rand::Rng; + Duration::from_secs(rand::thread_rng().gen_range(min_secs..=max_secs)) +} + +/// unix timestamp in milliseconds. Clamp to zero if the timestamp would +/// have been negative (prior to 1970) +fn now_unix_milli_ts() -> u64 { + let now = chrono::Utc::now().timestamp_millis(); + u64::try_from(now).unwrap_or(0) +} + +/// Cross-verify the director's announced targets against the config repo's +/// per-product delegation tree. +/// +/// Datadog's RC uses two TUF repos: the *director* has a flat top-level +/// `targets` signed by a single key, while the *config repo* authorizes +/// everything through per-product delegated roles (`APM_TRACING`, `ASM_DD`, …). +/// Trusting only the director collapses the model to one key, so every director +/// target must also be authorized — with the same `(length, hashes)` — by the +/// config delegation tree. +/// +/// We can't use the built-in `Database::target_description` walker because it +/// doesn't filter delegations by `paths` before checking `trusted_delegations`, +/// and its matcher handles only directory-prefix patterns, not the globs +/// Datadog uses (`datadog/*/APM_TRACING/*/*`). +/// +/// Assumes the config repo is flat (delegated roles are direct children of the +/// top-level targets); nested delegations would require making this recursive. +fn verify_director_against_config( + config_client: &TUFClient, + director_client: &TUFClient, +) -> anyhow::Result<()> { + let top_config_targets = config_client + .database() + .trusted_targets() + .ok_or_else(|| format_err!("config client has no trusted top-level targets"))?; + let trusted_delegations = config_client.database().trusted_delegations(); + + let director_targets = director_client + .database() + .trusted_targets() + .ok_or_else(|| format_err!("director client has no trusted targets"))?; + + // This operations is O(target * delegations) + // The path match is not that expensive, but this could be made faster by first building + // a perfix tree of the delegations, since the first segment of the paths are disjoints in + // the delegrations we receive from RC + for (path, dir_desc) in director_targets.targets() { + let cfg_desc = lookup_config_target(path, top_config_targets, trusted_delegations) + .ok_or_else(|| { + format_err!("director target {path} is not authorized by config delegations") + })?; + + if cfg_desc.length() != dir_desc.length() { + bail!( + "length mismatch between director and config for {path}: director={}, config={}", + dir_desc.length(), + cfg_desc.length() + ); + } + + // Check that the director and config hases sets are equal + if dir_desc.hashes() != cfg_desc.hashes() { + bail!("hash set mismatch between director and config for {path}"); + } + } + + Ok(()) +} + +/// Resolve `path` against the (flat) config delegation tree, returning the +/// target description from the first matching delegation that lists it. +/// Returns `None` if no delegation authorizes the path. +/// +/// TODO: make it compliant with Go-TUF & recursive delegations. +/// This currently only support "flat" delegations, as this is +/// the only thing we need for RC, but the TUF supports delegates +/// of delegates.s +fn lookup_config_target<'a>( + path: &TargetPath, + top: &'a tuf::verify::Verified, + trusted_delegations: &'a std::collections::HashMap< + MetadataPath, + tuf::verify::Verified, + >, +) -> Option<&'a TargetDescription> { + // Direct hit on the top-level targets (Datadog's are empty, but be safe). + if let direct @ Some(_) = top.targets().get(path) { + return direct; + } + + // Spec-style preorder walk over the (ordered) delegation list. + for delegation in top.delegations().roles() { + let matches_scope = delegation + .paths() + .iter() + .any(|pat| target_matches_pattern(path.as_str(), pat.as_str())); + if !matches_scope { + continue; + } + + if let Some(meta) = trusted_delegations.get(delegation.name()) { + if let Some(d) = meta.targets().get(path) { + return Some(d); + } + } + + // Scope matched but path not found: per TUF, a `terminating` delegation + // stops the search rather than falling through to siblings. + if delegation.terminating() { + return None; + } + } + + None +} + +/// TUF-style glob path matching. `*` matches any run of characters within a +/// single `/`-delimited segment; segments must otherwise match literally. +/// E.g. `datadog/*/APM_TRACING/*/*` matches +/// `datadog/556989/APM_TRACING//` but not +/// `datadog/x/y/APM_TRACING//`. +fn target_matches_pattern(path: &str, pattern: &str) -> bool { + let mut p_segs = pattern.split('/'); + let mut t_segs = path.split('/'); + loop { + match (p_segs.next(), t_segs.next()) { + (None, None) => return true, + (Some(p), Some(t)) if segment_matches(p, t) => continue, + _ => return false, + } + } +} + +/// Match a single path segment against a single pattern segment. `*` in the +/// pattern matches zero-or-more characters (within the segment, since segments +/// don't contain `/`). +fn segment_matches(pattern: &str, segment: &str) -> bool { + // Fast paths. + if pattern == "*" { + return true; + } + if !pattern.contains('*') { + return pattern == segment; + } + + // General case: literals split by `*`, anchored at both ends. + let parts: Vec<&str> = pattern.split('*').collect(); + let first = parts[0]; + if !segment.starts_with(first) { + return false; + } + let last = parts[parts.len() - 1]; + let mut cursor = &segment[first.len()..]; + // Middle literals must appear in order, non-overlapping. + for mid in &parts[1..parts.len() - 1] { + if mid.is_empty() { + continue; + } + match cursor.find(mid) { + Some(i) => cursor = &cursor[i + mid.len()..], + None => return false, + } + } + cursor.ends_with(last) && cursor.len() >= last.len() +} + +/// Return all currently trusted, unexpired targets. Targets that are expired or that lack a +/// supported hash algorithm are skipped with a debug log. +fn trusted_targets( + director_client: &TUFClient, +) -> anyhow::Result> + '_> { + Ok(director_client + .database() + .trusted_targets() + .ok_or_else(|| format_err!("missing targets from TUF director client"))? + .targets() + .iter() + .filter_map(|(path, desc)| { + TrustedTarget::try_create(path, desc) + .inspect_err(|e| { + debug!(%path, "Skipping target: error {}", e); + }) + .ok() + })) +} + +async fn store<'a, T>(repo: &mut TUFRepo, path: &MetadataPath, tms: T) -> anyhow::Result<()> +where + T: IntoIterator + 'a, +{ + for tm in tms { + repo.store_metadata( + path, + MetadataVersion::Number(tm.version), + &mut tm.raw.as_slice(), + ) + .await?; + } + Ok(()) +} + +async fn store_noversion( + repo: &mut TUFRepo, + path: &MetadataPath, + tms: &Option, +) -> anyhow::Result<()> { + if let Some(tm) = tms { + repo.store_metadata(path, MetadataVersion::None, &mut tm.raw.as_slice()) + .await?; + } + Ok(()) +} + +fn hash_algorithm_to_str(alg: &tuf::crypto::HashAlgorithm) -> &str { + match alg { + tuf::crypto::HashAlgorithm::Sha256 => "sha256", + tuf::crypto::HashAlgorithm::Sha512 => "sha512", + tuf::crypto::HashAlgorithm::Unknown(s) => s.as_str(), + _ => "unknown", + } +} + +/// Strip the leading `.` prefix from the basename of a TUF target path. +/// For instance "datadog/2///.config"` => `"datadog/2///config"` +/// +/// See https://datadoghq.atlassian.net/browse/RC-1859 for more information. +fn trim_hash_target_path(target_path: &str) -> anyhow::Result { + let (parent, basename) = target_path + .rsplit_once('/') + .ok_or_else(|| format_err!("invalid target: {target_path}"))?; + if basename.is_empty() { + bail!("invalid target: {target_path}") + } + + // Strip the leading `.` component if present. If the basename + // contains no `.`, keep it as-is (matches the previous behaviour). + let basename_trimmed = basename.split_once('.').map_or(basename, |(_, rest)| rest); + + Ok(format!("{parent}/{basename_trimmed}")) +} + +pub(crate) struct NewTarget { + pub path: crate::RemoteConfigPath, + pub version: u64, + /// Lowercase hex of the primary hash. + pub primary_hash: String, + /// All `(algorithm_name, hex_hash)` pairs for the target. + pub hashes: Vec<(String, String)>, + pub content: Vec, +} + +pub(crate) use cache::TargetCache; + +/// This module serves as a way to decouple the agentless logic from the rest of this crate +/// This is done for two purposes: +/// * Making review easier by allowing independent review of the RC checking alone +/// * Being able to isolate the agentless logic in it's own crate eventually so that we can reuse it +/// in bottlecap/ obs-pipeline without the rest of the code +mod cache { + use std::sync::{Arc, Mutex, MutexGuard}; + + use crate::{ + fetch::{ClientTargetRef, ConfigFetcherState, FileStorage, NewTarget, StoredTargetFile}, + RemoteConfigPath, + }; + use hashbrown::HashMap; + use libdd_capabilities::{HttpClientCapability, SleepCapability}; + use libdd_common::MutexExt as _; + use libdd_trace_protobuf::remoteconfig::{ConfigState, TargetFileHash, TargetFileMeta}; + + /// Wrapper over the shared [`ConfigFetcherState::target_files_by_path`] + /// map giving `AgentlessFetcher::apply` a set of small, self-contained + /// cache operations. Callers must first acquire the mutex via + /// [`Self::lock`] and then invoke the batch methods on the returned + /// guard, so the whole read cycle happens under a single lock scope. + pub(crate) struct TargetCache<'a, Storage: FileStorage> { + files: &'a Mutex, StoredTargetFile>>, + storage: &'a Storage, + expire_unused_files: bool, + } + + /// Locked view of the [`TargetCache`] — holds the mutex guard so + /// every method below runs under the same critical section, atomic + /// w.r.t. concurrent agentless fetchers sharing the same state. + /// + /// SAFETY / INVARIANT: never `.await` while this guard is alive — it + /// wraps a `std::sync::MutexGuard` and would block the runtime thread. + pub(crate) struct LockedTargetCache<'a, Storage: FileStorage> { + files: + MutexGuard<'a, HashMap, StoredTargetFile>>, + storage: &'a Storage, + expire_unused_files: bool, + } + + impl<'a, S: FileStorage> TargetCache<'a, S> { + pub(crate) fn new( + state: &'a ConfigFetcherState, + storage: &'a S, + ) -> Self { + TargetCache { + files: &state.target_files_by_path, + storage, + expire_unused_files: state.expire_unused_files, + } + } + + /// Acquire the cache lock. Hold the returned guard for the whole + /// read cycle so `is_cached_batch`, `store_batch`, `retain_only` + /// and `collect_handles` all run under one critical section. + pub(crate) fn lock(&self) -> LockedTargetCache<'a, S> { + LockedTargetCache { + files: self.files.lock_or_panic(), + storage: self.storage, + expire_unused_files: self.expire_unused_files, + } + } + } + + impl<'a, S: FileStorage> LockedTargetCache<'a, S> { + /// Returns the set of paths whose `(primary_hash, len)` already matches the cache. + pub(crate) fn is_cached_batch<'b>( + &self, + candidates: impl IntoIterator, + ) -> hashbrown::HashSet<&'b RemoteConfigPath> { + candidates + .into_iter() + .filter_map(|(parsed_path, primary_hash, len)| { + let stored = self.files.get(parsed_path)?; + // `stored.meta.length` is an `i64` coming from protobuf. A + // negative value should never appear, but if it does treat the + // entry as "not cached" so the caller re-fetches instead of + // matching a wrapped u64. + let stored_len = u64::try_from(stored.meta.length).ok()?; + (stored.hash == primary_hash && stored_len == len).then_some(parsed_path) + }) + .collect() + } + + pub(crate) fn store_batch( + &mut self, + targets: impl IntoIterator, + ) -> anyhow::Result<()> { + for NewTarget { + path, + version, + primary_hash, + hashes, + content, + } in targets + { + let length = i64::try_from(content.len()).map_err(|_| { + anyhow::format_err!( + "content length {} for path {} does not fit in i64", + content.len(), + path, + ) + })?; + let path: Arc = Arc::new(path); + let new_handle = if let Some(existing) = self.files.get(&path) { + self.storage + .update(&existing.handle, version, content) + .map(|()| existing.handle.clone())? + } else { + self.storage.store(version, path.clone(), content)? + }; + let file = StoredTargetFile { + hash: primary_hash, + state: ConfigState { + id: path.config_id().to_string(), + version, + product: path.product().to_string(), + apply_state: 2, // Acknowledged + apply_error: String::new(), + }, + meta: TargetFileMeta { + path: path.to_string(), + length, + hashes: hashes + .into_iter() + .map(|(algorithm, hash)| TargetFileHash { algorithm, hash }) + .collect(), + }, + handle: new_handle, + expiring: false, + }; + self.files.insert(path, file); + } + Ok(()) + } + + /// Evict every entry whose parsed TUF path is not in `active_paths`. No-op when + /// `expire_unused_files` is `false`. + pub(crate) fn retain_only(&mut self, active_paths: &hashbrown::HashSet<&RemoteConfigPath>) { + if !self.expire_unused_files { + return; + } + self.files.retain(|path, _| active_paths.contains(&**path)); + } + + /// Collect `Arc` handles for every target in `targets`, verifying + /// stored hash and length match, and marking each entry as non-expiring. + pub(crate) fn collect_handles( + &mut self, + targets: &[ClientTargetRef], + ) -> anyhow::Result>> { + let mut handles = Vec::with_capacity(targets.len()); + for target in targets { + let stored = self.files.get_mut(&target.path).ok_or_else(|| { + anyhow::format_err!( + "collect_handles: path {} not found in cache after fetch", + target.path + ) + })?; + let stored_len = u64::try_from(stored.meta.length).map_err(|_| { + anyhow::format_err!( + "collect_handles: cached length {} for {} is not a valid u64", + stored.meta.length, + target.path, + ) + })?; + if stored.hash != target.primary_hash || stored_len != target.length { + anyhow::bail!( + "collect_handles: cache mismatch for {}: stored hash={} len={}, \ + expected hash={} len={}", + target.path, + stored.hash, + stored.meta.length, + target.primary_hash, + target.length, + ); + } + stored.expiring = false; + handles.push(stored.handle.clone()); + } + Ok(handles) + } + } +} + +// ── Debug helpers: render `raw: Vec` fields as JSON ──────────────────── + +struct RawJson<'a>(&'a [u8]); + +impl fmt::Debug for RawJson<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let RawJson(bytes) = self; + match serde_json::from_slice::(bytes) { + Ok(v) => write!(f, "{v:#}"), + Err(_) => write!(f, "<{} non-JSON bytes>", bytes.len()), + } + } +} + +struct DebugTopMeta<'a>(&'a remoteconfig::TopMeta); + +impl fmt::Debug for DebugTopMeta<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::TopMeta { version, raw } = self.0; + f.debug_struct("TopMeta") + .field("version", version) + .field("raw", &RawJson(raw)) + .finish() + } +} + +struct DebugDelegatedMeta<'a>(&'a remoteconfig::DelegatedMeta); + +impl fmt::Debug for DebugDelegatedMeta<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::DelegatedMeta { version, role, raw } = self.0; + f.debug_struct("DelegatedMeta") + .field("version", version) + .field("role", role) + .field("raw", &RawJson(raw)) + .finish() + } +} + +struct DebugFile<'a>(&'a remoteconfig::File); + +impl fmt::Debug for DebugFile<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::File { path, raw } = self.0; + f.debug_struct("File") + .field("path", path) + .field("raw", &RawJson(raw)) + .finish() + } +} + +struct DebugConfigMetas<'a>(&'a remoteconfig::ConfigMetas); + +impl fmt::Debug for DebugConfigMetas<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::ConfigMetas { + roots, + timestamp, + snapshot, + top_targets, + delegated_targets, + } = self.0; + f.debug_struct("ConfigMetas") + .field("roots", &roots.iter().map(DebugTopMeta).collect::>()) + .field("timestamp", ×tamp.as_ref().map(DebugTopMeta)) + .field("snapshot", &snapshot.as_ref().map(DebugTopMeta)) + .field("top_targets", &top_targets.as_ref().map(DebugTopMeta)) + .field( + "delegated_targets", + &delegated_targets + .iter() + .map(DebugDelegatedMeta) + .collect::>(), + ) + .finish() + } +} + +struct DebugDirectorMetas<'a>(&'a remoteconfig::DirectorMetas); + +impl fmt::Debug for DebugDirectorMetas<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::DirectorMetas { + roots, + timestamp, + snapshot, + targets, + } = &self.0; + f.debug_struct("DirectorMetas") + .field("roots", &roots.iter().map(DebugTopMeta).collect::>()) + .field("timestamp", ×tamp.as_ref().map(DebugTopMeta)) + .field("snapshot", &snapshot.as_ref().map(DebugTopMeta)) + .field("targets", &targets.as_ref().map(DebugTopMeta)) + .finish() + } +} + +struct DebugLatestConfigsResponse<'a>(&'a remoteconfig::LatestConfigsResponse); + +impl fmt::Debug for DebugLatestConfigsResponse<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let remoteconfig::LatestConfigsResponse { + config_metas, + director_metas, + target_files, + } = &self.0; + f.debug_struct("LatestConfigsResponse") + .field("config_metas", &config_metas.as_ref().map(DebugConfigMetas)) + .field( + "director_metas", + &director_metas.as_ref().map(DebugDirectorMetas), + ) + .field( + "target_files", + &target_files.iter().map(DebugFile).collect::>(), + ) + .finish() + } +} + +/// Returns a value that implements [`fmt::Debug`] for [`remoteconfig::LatestConfigsResponse`], +/// rendering every `raw` byte field as a parsed JSON value instead of a raw byte array. +pub fn debug_latest_configs_response( + resp: &remoteconfig::LatestConfigsResponse, +) -> impl fmt::Debug + '_ { + DebugLatestConfigsResponse(resp) +} + +#[cfg(test)] +mod tests { + use libdd_common::Endpoint; + + use crate::fetch::AgentlessConfig; + use crate::fetch::NativeAgentlessFetcher; + + use super::trim_hash_target_path; + use super::Site; + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn test_create_fetcher_for_site() { + for site in ["datad0g.com", "datadoghq.com", "ddog-gov.com"] { + let endpoint = Endpoint::agentless(site, "abc".to_string()).unwrap(); + let cfg = AgentlessConfig::new("hostname".to_string(), &endpoint).unwrap(); + NativeAgentlessFetcher::new( + cfg, + endpoint, + libdd_capabilities_impl::NativeHttpClient::new_without_connection_pooling(), + ) + .await + .unwrap_or_else(|e| panic!("failed to instantiate fetcher for site {site}: {e}")); + } + } + + #[test] + #[cfg_attr(miri, ignore)] + fn strips_hash_prefix() { + assert_eq!( + trim_hash_target_path("datadog/2/APM_TRACING/abcd/deadbeef.config").unwrap(), + "datadog/2/APM_TRACING/abcd/config" + ); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn no_hash_prefix_is_kept() { + assert_eq!( + trim_hash_target_path("datadog/2/APM_TRACING/abcd/config").unwrap(), + "datadog/2/APM_TRACING/abcd/config" + ); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn backslash_is_not_a_separator() { + // Windows-style separators must NOT be treated as path separators. + // The whole string is the basename here. + assert!(trim_hash_target_path(r"datadog\2\foo.bar").is_err()); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn empty_or_no_slash_is_error() { + assert!(trim_hash_target_path("").is_err()); + assert!(trim_hash_target_path("deadbeef.config").is_err()); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn trailing_slash_is_error() { + assert!(trim_hash_target_path("datadog/2/foo/").is_err()); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agentless_config_new_rejects_empty_hostname() { + let endpoint = Endpoint::agentless("datadoghq.com", "abc".to_string()).unwrap(); + assert!(matches!( + AgentlessConfig::new(String::new(), &endpoint), + Err(super::AgentlessConfigError::EmptyHostname) + )); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agentless_config_new_rejects_endpoint_without_api_key() { + // Endpoint constructed via `from_slice` has no `api_key`, which is one + // of the required agentless preconditions. + let endpoint = Endpoint::from_slice("https://datadoghq.com/"); + assert!(matches!( + AgentlessConfig::new("host".to_string(), &endpoint), + Err(super::AgentlessConfigError::InvalidEndpoint) + )); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agentless_config_new_rejects_non_https_endpoint() { + let mut endpoint = Endpoint::from_slice("http://datadoghq.com/"); + endpoint.api_key = Some("abc".to_string().into()); + assert!(matches!( + AgentlessConfig::new("host".to_string(), &endpoint), + Err(super::AgentlessConfigError::InvalidEndpoint) + )); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agentless_config_new_accepts_well_formed_config() { + let endpoint = Endpoint::agentless("datadoghq.com", "abc".to_string()).unwrap(); + let cfg = AgentlessConfig::new("host".to_string(), &endpoint).expect("well-formed"); + assert_eq!(cfg.hostname(), "host"); + // Endpoint has been rewritten to `config.` and kept the api key. + assert!(cfg + .agentless_endpoint() + .url + .host() + .is_some_and(|h| h.starts_with("config."))); + assert!(cfg.agentless_endpoint().api_key.is_some()); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn agentless_config_builders() { + let endpoint = Endpoint::agentless("datadoghq.com", "abc".to_string()).unwrap(); + let cfg = AgentlessConfig::new("host".to_string(), &endpoint) + .unwrap() + .with_agent_uuid("uuid-1".to_string()); + assert_eq!(cfg.agent_uuid(), Some("uuid-1")); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_compute_backoff() { + use super::compute_backoff; + use std::time::Duration; + + assert_eq!(compute_backoff(0), None); + + let b1 = compute_backoff(1).unwrap(); + assert!((Duration::from_secs(30)..=Duration::from_secs(60)).contains(&b1)); + + let b2 = compute_backoff(2).unwrap(); + assert!((Duration::from_secs(60)..=Duration::from_secs(90)).contains(&b2)); + + let b3 = compute_backoff(3).unwrap(); + assert!((Duration::from_secs(90)..=Duration::from_secs(120)).contains(&b3)); + + let b42 = compute_backoff(42).unwrap(); + assert!((Duration::from_secs(90)..=Duration::from_secs(120)).contains(&b42)); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_target_matches_pattern() { + use super::target_matches_pattern as m; + + // Real Datadog delegation pattern. + assert!(m( + "datadog/556989/APM_TRACING/abc/def", + "datadog/*/APM_TRACING/*/*" + )); + + // Wrong product segment. + assert!(!m( + "datadog/556989/ASM_DD/abc/def", + "datadog/*/APM_TRACING/*/*" + )); + + // Extra path segment must not match (`*` doesn't cross `/`). + assert!(!m( + "datadog/x/y/APM_TRACING/abc/def", + "datadog/*/APM_TRACING/*/*" + )); + + // Missing path segment. + assert!(!m( + "datadog/556989/APM_TRACING/abc", + "datadog/*/APM_TRACING/*/*" + )); + + // Employee-prefix delegation. + assert!(m("employee/ASM_DD/abc/def", "employee/ASM_DD/*/*")); + assert!(!m("employee/CWS_DD/abc/def", "employee/ASM_DD/*/*")); + + // Partial-segment wildcards. + assert!(super::segment_matches("foo*bar", "foo123bar")); + assert!(super::segment_matches("foo*", "foobar")); + assert!(super::segment_matches("*bar", "foobar")); + assert!(super::segment_matches( + "ba*bar k*g of the *elepha*ts*", + "babar king of the elephants" + )); + assert!(!super::segment_matches("foo*bar", "fobar")); + assert!(!super::segment_matches(" *foobar**", "fobar")); + // `*` does not cross `/`. + assert!(!m("foo/bar", "foo*bar")); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_site_from_host() { + assert_eq!(Site::from_host("config.datadoghq.com"), Site::Prod); + assert_eq!(Site::from_host("config.us3.datadoghq.com"), Site::Prod); + assert_eq!(Site::from_host("config.datadoghq.eu"), Site::Prod); + assert_eq!(Site::from_host("config.datad0g.com"), Site::Staging); + assert_eq!(Site::from_host("datad0g.com"), Site::Staging); + assert_eq!(Site::from_host("config.ddog-gov.com"), Site::Gov); + assert_eq!(Site::from_host("config.foo.ddog-gov.com"), Site::Gov); + } +} + +#[cfg(test)] +mod integration_tests; diff --git a/libdd-remote-config/src/fetch/agentless/integration_tests.rs b/libdd-remote-config/src/fetch/agentless/integration_tests.rs new file mode 100644 index 0000000000..148cc1fb40 --- /dev/null +++ b/libdd-remote-config/src/fetch/agentless/integration_tests.rs @@ -0,0 +1,1262 @@ +// Copyright 2026-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! Mirrors the datadog-agent uptane +//! `client_test.go` harness (generate signed config + director repos, feed a +//! `LatestConfigsResponse` to the client), but drives libdatadog's +//! `fetch_config`/`apply` path through a mock HTTP capability. +//! +//! These tests exercise root rotation, various input shapes, and +//! target-integrity failure modes. +//! +//! # Scenarios intentionally NOT covered here +//! +//! Some failure modes exercised by the agent's uptane test suite cannot be +//! reproduced meaningfully in this in-memory harness and are therefore left +//! out on purpose. If the underlying assumptions change (e.g. we move to a +//! persistent TUF backend, or expose mutation APIs on the trusted database), +//! these must be revisited: +//! +//! * **On-disk or trusted-database tampering**: the rust-tuf `Database` used by the fetcher is +//! in-memory and offers no public mutation API, so we cannot flip a byte inside an +//! already-verified snapshot/targets record and observe how `apply()` reacts. Any regression here +//! would have to be caught inside rust-tuf itself. +//! * **Persisted-cache tampering across restarts**: the fetcher does not own a persistent cache. +//! The on-disk representation belongs to the embedding host (`FileStorage` implementation). Tests +//! here use `NoopStorage`, so anything a hostile filesystem could do between two process +//! lifetimes (swap files, roll back versions, corrupt bytes) is out of scope and must be covered +//! by the storage implementation's own tests. +//! * **Transport-level integrity and MITM**: TLS and endpoint authentication are provided by the +//! `HttpClientCapability` implementation. The mock HTTP capability short-circuits the wire, so +//! replay, downgrade or certificate-swap attacks are not exercised here. +//! * **Signature forgery or weak-key acceptance**: signing keys are generated fresh in each test, +//! so we cannot meaningfully test "attacker with the old key" scenarios. Those are the +//! responsibility of rust-tuf's own test suite. +#![allow(clippy::unwrap_used)] + +use super::*; +use crate::fetch::{ConfigFetcherState, ConfigInvariants, FileStorage}; +use crate::RemoteConfigPath; +use libdd_capabilities::http::{HttpClientCapability, HttpError}; +use libdd_capabilities::maybe_send::MaybeSend; +use std::collections::VecDeque; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use tuf::crypto::{Ed25519PrivateKey, HashAlgorithm, PrivateKey}; +use tuf::database::Database; +use tuf::interchange::Json; +use tuf::metadata::{ + Delegation, MetadataDescription, MetadataPath, RawSignedMetadataSet, SignedMetadata, + SnapshotMetadata, TargetPath, TargetsMetadataBuilder, TimestampMetadataBuilder, +}; +use tuf::repo_builder::RepoBuilder; +use tuf::repository::EphemeralRepository; + +// ---- mock HTTP capability ------------------------------------------------ + +#[derive(Clone, Debug)] +struct MockHttp { + responses: Arc>>>, + requests: Arc>>, + /// UUID returned for GET /api/v0.1/org. Some tests want to change it mid-run + /// (e.g. serve a mismatching UUID) so this is a Mutex, not an atomic snapshot. + org_uuid: Arc>, + /// Count of GET /api/v0.1/org requests observed, used to assert the + /// concurrent prefetch (poll 1) and the lazy re-fetch (post root-rotation). + org_requests: Arc>, +} + +impl MockHttp { + fn new() -> Self { + Self { + responses: Arc::new(Mutex::new(VecDeque::new())), + requests: Arc::new(Mutex::new(Vec::new())), + org_uuid: Arc::new(Mutex::new(String::new())), + org_requests: Arc::new(Mutex::new(0)), + } + } + + fn push(&self, resp: &remoteconfig::LatestConfigsResponse) { + self.responses + .lock() + .unwrap() + .push_back(resp.encode_to_vec()); + } + + fn request_at(&self, i: usize) -> remoteconfig::LatestConfigsRequest { + self.requests.lock().unwrap()[i].clone() + } + + fn set_org_uuid(&self, uuid: &str) { + *self.org_uuid.lock().unwrap() = uuid.to_string(); + } + + fn org_request_count(&self) -> u32 { + *self.org_requests.lock().unwrap() + } +} + +impl HttpClientCapability for MockHttp { + fn new_client() -> Self { + Self::new() + } + + fn new_without_connection_pooling() -> Self { + Self::new() + } + + #[allow(clippy::manual_async_fn)] + fn request( + &self, + req: http::Request, + ) -> impl Future, HttpError>> + MaybeSend { + let responses = self.responses.clone(); + let requests = self.requests.clone(); + let org_uuid = self.org_uuid.clone(); + let org_requests = self.org_requests.clone(); + // Capture the path off the request before moving the body. + let path = req.uri().path().to_owned(); + async move { + if path == "/api/v0.1/org" { + *org_requests.lock().unwrap() += 1; + let resp = remoteconfig::OrgDataResponse { + uuid: org_uuid.lock().unwrap().clone(), + }; + return Ok(http::Response::builder() + .status(200) + .body(Bytes::from(resp.encode_to_vec())) + .unwrap()); + } + let body = req.into_body(); + if let Ok(parsed) = remoteconfig::LatestConfigsRequest::decode(body) { + requests.lock().unwrap().push(parsed); + } + let bytes = responses + .lock() + .unwrap() + .pop_front() + .expect("mock http: no queued response"); + Ok(http::Response::builder() + .status(200) + .body(Bytes::from(bytes)) + .unwrap()) + } + } +} + +// ---- no-op file storage -------------------------------------------------- + +#[derive(Default)] +struct NoopStorage; + +impl FileStorage for NoopStorage { + type StoredFile = (); + + fn store( + &self, + _version: u64, + _path: Arc, + _contents: Vec, + ) -> anyhow::Result> { + Ok(Arc::new(())) + } + + fn update( + &self, + _file: &Arc, + _version: u64, + _contents: Vec, + ) -> anyhow::Result<()> { + Ok(()) + } +} + +fn test_state() -> ConfigFetcherState<(), libdd_capabilities_impl::NativeCapabilities> { + ConfigFetcherState::with_client( + ConfigInvariants { + language: "test".to_string(), + tracer_version: "0.0.0".to_string(), + endpoint: Endpoint::from_slice("http://localhost/"), + agentless: None, + }, + libdd_capabilities_impl::NativeCapabilities::new_client(), + ) +} + +// ---- TUF repo generation (mirrors uptane client_test.go) ----------------- + +fn new_key() -> Ed25519PrivateKey { + Ed25519PrivateKey::from_pkcs8(&Ed25519PrivateKey::pkcs8().unwrap()).unwrap() +} + +/// Build a fresh v1 repo (root/targets/snapshot/timestamp all v1, empty +/// targets, consistent_snapshot=true) signed entirely by `key`. +async fn build_v1(key: &Ed25519PrivateKey) -> RawSignedMetadataSet { + let mut repo = EphemeralRepository::::new(); + RepoBuilder::create(&mut repo) + .trusted_root_keys(&[key]) + .trusted_targets_keys(&[key]) + .trusted_snapshot_keys(&[key]) + .trusted_timestamp_keys(&[key]) + .commit() + .await + .unwrap() +} + +/// Rotate only the root (v1 -> v2), keeping the same keys. rust-tuf's +/// `update_root` purges all non-root trusted metadata on this bump, which is +/// what triggers the top-targets re-fetch the stuck test exercises. +async fn rotate_root( + key: &Ed25519PrivateKey, + prev: &RawSignedMetadataSet, +) -> RawSignedMetadataSet { + let db = Database::::from_trusted_metadata(prev).unwrap(); + let mut repo = EphemeralRepository::::new(); + RepoBuilder::from_database(&mut repo, &db) + .trusted_root_keys(&[key]) + .trusted_targets_keys(&[key]) + .trusted_snapshot_keys(&[key]) + .trusted_timestamp_keys(&[key]) + .stage_root() + .unwrap() + .commit() + .await + .unwrap() +} + +fn meta_version(raw: &[u8]) -> u64 { + let v: serde_json::Value = serde_json::from_slice(raw).unwrap(); + v["signed"]["version"].as_u64().unwrap() +} + +fn top(raw: &[u8]) -> remoteconfig::TopMeta { + remoteconfig::TopMeta { + version: meta_version(raw), + raw: raw.to_vec(), + } +} + +fn director_metas(set: &RawSignedMetadataSet) -> remoteconfig::DirectorMetas { + remoteconfig::DirectorMetas { + roots: vec![top(set.root().unwrap().as_bytes())], + timestamp: Some(top(set.timestamp().unwrap().as_bytes())), + snapshot: Some(top(set.snapshot().unwrap().as_bytes())), + targets: Some(top(set.targets().unwrap().as_bytes())), + } +} + +/// Build a `LatestConfigsResponse` from raw config metadata plus a director set. +fn response( + config_roots: &[&[u8]], + config_timestamp: &[u8], + config_snapshot: &[u8], + config_top_targets: Option<&[u8]>, + delegated: Vec, + director: &RawSignedMetadataSet, +) -> remoteconfig::LatestConfigsResponse { + remoteconfig::LatestConfigsResponse { + config_metas: Some(remoteconfig::ConfigMetas { + roots: config_roots.iter().map(|r| top(r)).collect(), + timestamp: Some(top(config_timestamp)), + snapshot: Some(top(config_snapshot)), + top_targets: config_top_targets.map(top), + delegated_targets: delegated, + }), + director_metas: Some(director_metas(director)), + target_files: vec![], + } +} + +/// Construct a fetcher wired to a mock HTTP client and pinned to the given +/// root bytes (bypassing `AgentlessFetcher::new`, whose `C::new_client()` +/// would discard our pre-seeded mock). +async fn fetcher( + http: MockHttp, + config_root: Vec, + director_root: Vec, +) -> AgentlessFetcher { + AgentlessFetcher { + endpoint: Endpoint { + timeout_ms: 30_000, + ..Endpoint::from_slice("http://localhost/") + }, + http, + director_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(director_root.clone()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await + .unwrap(), + config_client: TUFClient::with_trusted_root( + tuf::client::Config::default(), + &RawSignedMetadata::new(config_root.clone()), + TUFRepo::new(), + TUFRepo::new(), + ) + .await + .unwrap(), + last_config_top_targets: None, + org_uuid: None, + org_data_prefetched: false, + hostname: "test-host".to_string(), + agent_uuid_override: Some("test-uuid".to_string()), + products: HashSet::new(), + opaque_backend_state: Vec::new(), + refresh_interval: Duration::from_secs(60), + consecutive_failures: 0, + } +} + +fn dummy_client() -> remoteconfig::Client { + remoteconfig::Client { + products: vec!["APM_TRACING".to_string()], + ..Default::default() + } +} + +fn config_root_version(f: &AgentlessFetcher) -> u64 { + f.config_client.database().trusted_root().version() +} + +fn config_snapshot_version(f: &AgentlessFetcher) -> Option { + f.config_client + .database() + .trusted_snapshot() + .map(|s| s.version()) +} + +// ---- tests --------------------------------------------------------------- + +/// incident-45734: a config **root rotation** where the backend omits the +/// (unchanged) top-targets must still converge. Before the fix the wipe drops +/// the top-targets and `update()` is stuck; the cache re-serves them. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_root_rotation_without_top_targets_still_converges() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let cfg2 = rotate_root(&config_key, &cfg1).await; // config root v1 -> v2 + let dir1 = build_v1(&director_key).await; + + let http = MockHttp::new(); + // Poll 1: full config metadata. + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + cfg1.timestamp().unwrap().as_bytes(), + cfg1.snapshot().unwrap().as_bytes(), + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + // Poll 2: config ROOT rotated (v2), top-targets version unchanged so the + // backend sends NONE; reuse the v1 timestamp/snapshot. + http.push(&response( + &[cfg2.root().unwrap().as_bytes()], + cfg1.timestamp().unwrap().as_bytes(), + cfg1.snapshot().unwrap().as_bytes(), + None, + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + // Poll 1 succeeds and advances the config DB to root v1 / snapshot v1. + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + + // Poll 2 (root rotation, no top-targets) must still converge. + f.fetch_config(dummy_client(), &cache) + .await + .expect("root rotation with omitted top-targets must converge"); + assert_eq!(config_root_version(&f), 2); + + // Step 10: reported versions always match the live trusted DB. + let req1 = http.request_at(0); + assert_eq!(req1.current_config_snapshot_version, 0); + assert_eq!(req1.current_config_root_version, 1); + assert_eq!(req1.current_director_root_version, 1); + let req2 = http.request_at(1); + // After poll 1 the DB advanced: snapshot v1, config root still v1. + assert_eq!( + req2.current_config_snapshot_version, + meta_version(cfg1.snapshot().unwrap().as_bytes()) + ); + assert_eq!( + req2.current_config_root_version, + meta_version(cfg1.root().unwrap().as_bytes()) + ); +} + +/// D-F1: an `apply()` that fails *after* advancing the config trusted DB must +/// leave the fetcher recoverable. The reset rebuilds the clients from the +/// pinned roots, so the next poll reports the clean (embedded) versions and +/// converges — no stuck from a partially-advanced trusted DB. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_apply_error_resets_and_recovers() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let cfg2 = rotate_root(&config_key, &cfg1).await; + let dir1 = build_v1(&director_key).await; + + let good = |top_targets: Option<&[u8]>, roots: &[&[u8]]| { + response( + roots, + cfg1.timestamp().unwrap().as_bytes(), + cfg1.snapshot().unwrap().as_bytes(), + top_targets, + vec![], + &dir1, + ) + }; + + let http = MockHttp::new(); + // Poll 1: good, advances to config root v1. + http.push(&good( + Some(cfg1.targets().unwrap().as_bytes()), + &[cfg1.root().unwrap().as_bytes()], + )); + // Poll 2: config root rotates to v2 (config update succeeds and advances + // the trusted root), then a garbage delegated-targets blob makes apply() + // fail *after* the advance. + let mut bad = good( + Some(cfg1.targets().unwrap().as_bytes()), + &[cfg2.root().unwrap().as_bytes()], + ); + bad.config_metas.as_mut().unwrap().delegated_targets = vec![remoteconfig::DelegatedMeta { + version: 1, + role: "APM_TRACING".to_string(), + raw: b"not valid tuf metadata".to_vec(), + }]; + http.push(&bad); + // Poll 3: good again — must recover. + http.push(&good( + Some(cfg1.targets().unwrap().as_bytes()), + &[cfg1.root().unwrap().as_bytes()], + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + // Poll 1 ok. + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + assert_eq!(config_snapshot_version(&f), Some(1)); + // A fully successful poll leaves the backoff counter at 0. + assert_eq!(f.consecutive_failures(), 0); + assert_eq!(f.next_backoff(), None); + + // Poll 2 fails (after the config root advanced to v2) and resets. + assert!(f.fetch_config(dummy_client(), &cache).await.is_err()); + // Reset purges snapshot/targets/timestamp/delegations but preserves the + // trusted root that `update()` already advanced to v2 — recovery does not + // roll back the trust chain to the embedded root v1. + assert_eq!(config_root_version(&f), 2); + assert_eq!(config_snapshot_version(&f), None); + assert!(f.opaque_backend_state.is_empty()); + assert!(f.products.is_empty()); + assert!(f.last_config_top_targets.is_none()); + // E-1: a successful HTTP fetch that fails `apply()` (verification) must still + // count as a failed poll, so the client backs off during sustained + // verification failures instead of hot-looping. + assert_eq!(f.consecutive_failures(), 1); + + // Poll 3 recovers. The response's `cfg1.root` (v1) is older than the + // preserved trusted root v2 and is ignored by `update_root`; snapshot, + // timestamp and targets are re-signed by the same key that root v2 trusts, + // so they verify and repopulate the purged non-root state. + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 2); + assert_eq!(config_snapshot_version(&f), Some(1)); + // A fully successful poll clears the counter again. + assert_eq!(f.consecutive_failures(), 0); + assert_eq!(f.next_backoff(), None); + + // The post-reset poll reported the *preserved* trusted root version (v2) + // and the cleared snapshot version (0), so the backend does not have to + // re-send the root rotation chain. + let req3 = http.request_at(2); + assert_eq!(req3.current_config_snapshot_version, 0); + assert_eq!(req3.current_config_root_version, 2); + assert_eq!(req3.current_director_root_version, 1); +} + +/// Build a config repo (v1) that authorizes both `known_path` and +/// `unknown_path` through a single delegated role `role_name` whose glob +/// `paths` cover both products. Returns the signed config metadata set plus the +/// raw delegated-targets blob (to feed as `DelegatedMeta`). +async fn build_config_with_delegation( + config_key: &Ed25519PrivateKey, + product_key: &Ed25519PrivateKey, + role_name: &str, + entries: &[(&str, &[u8])], + glob_paths: &[&str], + target_hashes: &[HashAlgorithm], +) -> (RawSignedMetadataSet, Vec) { + // Delegated targets blob: describes every authorized (path, content). + let mut builder = TargetsMetadataBuilder::new(); + for (path, content) in entries { + builder = builder + .insert_target_from_slice( + TargetPath::new((*path).to_string()).unwrap(), + content, + target_hashes, + ) + .unwrap(); + } + let delegated = builder.signed::(product_key).unwrap(); + let raw_delegated = delegated.to_raw().unwrap().as_bytes().to_vec(); + + // Top-level config targets: delegate the glob paths to `role_name`. + let mut delegation = Delegation::builder(MetadataPath::new(role_name.to_string()).unwrap()) + .key(product_key.public()); + for g in glob_paths { + delegation = delegation.delegate_path(TargetPath::new((*g).to_string()).unwrap()); + } + let delegation = delegation.build().unwrap(); + + let role_path = MetadataPath::new(role_name.to_string()).unwrap(); + let delegated_desc = + MetadataDescription::from_slice(&raw_delegated, 1, &[HashAlgorithm::Sha256]).unwrap(); + + let mut repo = EphemeralRepository::::new(); + let set = RepoBuilder::create(&mut repo) + .trusted_root_keys(&[config_key]) + .trusted_targets_keys(&[config_key]) + .trusted_snapshot_keys(&[config_key]) + .trusted_timestamp_keys(&[config_key]) + .stage_root() + .unwrap() + .add_delegation_key(product_key.public().clone()) + .add_delegation_role(delegation) + .stage_targets() + .unwrap() + .stage_snapshot_with_builder(|builder| { + builder.insert_metadata_description(role_path.clone(), delegated_desc.clone()) + }) + .unwrap() + .commit() + .await + .unwrap(); + + (set, raw_delegated) +} + +/// Build a director repo (v1) that announces every `(path, content)` entry as a +/// top-level target (sha256), matching the config authorization. +async fn build_director_with_targets( + director_key: &Ed25519PrivateKey, + entries: &[(&str, &[u8])], + target_hashes: &[HashAlgorithm], +) -> RawSignedMetadataSet { + let mut repo = EphemeralRepository::::new(); + let mut builder = RepoBuilder::create(&mut repo) + .trusted_root_keys(&[director_key]) + .trusted_targets_keys(&[director_key]) + .trusted_snapshot_keys(&[director_key]) + .trusted_timestamp_keys(&[director_key]) + .stage_root_if_necessary() + .unwrap() + .target_hash_algorithms(target_hashes); + for (path, content) in entries { + builder = builder + .add_target( + TargetPath::new((*path).to_string()).unwrap(), + futures_util::io::Cursor::new(content.to_vec()), + ) + .await + .unwrap(); + } + builder.stage_targets().unwrap().commit().await.unwrap() +} + +/// E-F1 / G-F1: a director target for a product the closed `RemoteConfigProduct` +/// enum does not know must not fail the fetch of the other, known targets. +/// +/// The cache owns the parsing rules (`TargetCache::is_parseable_path`); `apply()` +/// consults it to drop unparseable/unknown-product targets before they reach +/// `active_targets`, so `collect_handles` never sees a path it can't serve. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_unknown_product_target_is_not_stuck_known_targets() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let known_path = "datadog/2/APM_TRACING/cfgid/config"; + let unknown_path = "datadog/2/BRAND_NEW_PRODUCT/cfgid/config"; + let known_content: &[u8] = b"known apm config payload"; + let unknown_content: &[u8] = b"brand new product payload"; + + let entries: &[(&str, &[u8])] = &[(known_path, known_content), (unknown_path, unknown_content)]; + + // Config authorizes BOTH paths (so `verify_director_against_config` passes); + // the divergence is purely that libdatadog's product enum can't parse the + // second one. + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + entries, + &[ + "datadog/*/APM_TRACING/*/*", + "datadog/*/BRAND_NEW_PRODUCT/*/*", + ], + &[HashAlgorithm::Sha256], + ) + .await; + let dir = build_director_with_targets(&director_key, entries, &[HashAlgorithm::Sha256]).await; + + let resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, entries); + + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let res = f + .fetch_config(dummy_client(), &cache) + .await + .expect("a config-authorized unknown-product target must not stuck the fetch"); + + let returned: Vec = res.targets.iter().map(|t| t.path.to_string()).collect(); + assert_eq!( + returned, + vec![known_path.to_string()], + "only the known-product target should be active" + ); + + // The active batch is fully servable: `collect_handles` succeeds (no stuck). + let handles = cache + .lock() + .collect_handles(&res.targets) + .expect("active batch must not stuck collect_handles"); + assert_eq!(handles.len(), 1, "only the known target should be served"); +} + +/// Assemble a `LatestConfigsResponse` from a config set + its raw delegated +/// blob, a director set, and the `(path, content)` entries served as files. +fn delegated_response( + cfg: &RawSignedMetadataSet, + raw_delegated: Vec, + role_name: &str, + dir: &RawSignedMetadataSet, + entries: &[(&str, &[u8])], +) -> remoteconfig::LatestConfigsResponse { + remoteconfig::LatestConfigsResponse { + config_metas: Some(remoteconfig::ConfigMetas { + roots: vec![top(cfg.root().unwrap().as_bytes())], + timestamp: Some(top(cfg.timestamp().unwrap().as_bytes())), + snapshot: Some(top(cfg.snapshot().unwrap().as_bytes())), + top_targets: Some(top(cfg.targets().unwrap().as_bytes())), + delegated_targets: vec![remoteconfig::DelegatedMeta { + version: meta_version(&raw_delegated), + role: role_name.to_string(), + raw: raw_delegated, + }], + }), + director_metas: Some(director_metas(dir)), + target_files: entries + .iter() + .map(|(path, content)| remoteconfig::File { + path: (*path).to_string(), + raw: content.to_vec(), + }) + .collect(), + } +} + +/// A-F1 (libdd #14): `verify_director_against_config` must require the director +/// and config hash sets to be *exactly equal*. Here the director publishes +/// sha256+sha512 for a target while config pins only sha256. The old +/// overlap-only check (shared algos agree) would accept this — letting the +/// director assert an arbitrary sha512 digest config never authorized — so the +/// whole `apply()` must now fail. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_director_hash_superset_is_rejected() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let path = "datadog/2/APM_TRACING/cfgid/config"; + let content: &[u8] = b"apm config payload"; + let entries: &[(&str, &[u8])] = &[(path, content)]; + + // Config pins sha256 only. + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + entries, + &["datadog/*/APM_TRACING/*/*"], + &[HashAlgorithm::Sha256], + ) + .await; + // Director publishes a superset: sha256 + sha512 (digests still correct for + // the content, so only the *set* differs). + let dir = build_director_with_targets( + &director_key, + entries, + &[HashAlgorithm::Sha256, HashAlgorithm::Sha512], + ) + .await; + + let resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, entries); + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("director hash superset must be rejected (exact-equality)"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("hash set mismatch"), + "expected a hash-set mismatch error, got: {msg}" + ); +} + +// ---- target-file integrity tests ----------------------------------------- +// +// These tests confirm that byte- and metadata-level divergences between the +// wire response, the director metadata, and the config-repo delegations are +// all rejected before any target reaches the cache. They complement the +// hash-set superset test above. + +/// Convenience: build a matched (config, director) pair for a single +/// APM_TRACING target and return the pieces the tests below tweak. +async fn matched_pair( + path: &'static str, + content: &'static [u8], +) -> ( + RawSignedMetadataSet, + Vec, + RawSignedMetadataSet, +) { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let entries: &[(&str, &[u8])] = &[(path, content)]; + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + entries, + &["datadog/*/APM_TRACING/*/*"], + &[HashAlgorithm::Sha256], + ) + .await; + let dir = build_director_with_targets(&director_key, entries, &[HashAlgorithm::Sha256]).await; + + (cfg, raw_delegated, dir) +} + +/// Feed a `LatestConfigsResponse` in which the raw bytes served for a target +/// no longer hash to what the director signed. Same length — so the length +/// check passes and the hash check is the one that fires. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_tampered_target_bytes_are_rejected() { + let path = "datadog/2/APM_TRACING/cfgid/config"; + let content: &[u8] = b"apm config payload"; + let (cfg, raw_delegated, dir) = matched_pair(path, content).await; + + let entries: &[(&str, &[u8])] = &[(path, content)]; + let mut resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, entries); + // Same length, different bytes: the director metadata signed the SHA-256 + // of `content`, so any equal-length flip fails the hash check. + // (`content` must be non-empty, otherwise the XOR flip below is a no-op.) + let tampered: Vec = content.iter().map(|b| b ^ 0x01).collect(); + assert_eq!(tampered.len(), content.len()); + resp.target_files[0].raw = tampered; + + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("tampered target bytes must be rejected"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("hash did not match"), + "expected a hash-mismatch error on tampered bytes, got: {msg}" + ); + + // Nothing must have been stored. + assert!(state.target_files_by_path.lock().unwrap().is_empty()); +} + +/// Feed a response whose raw bytes for a target have the wrong length. +/// Because the length check runs before the hash check in `fetch_target`, +/// this is the specific error we expect — not a hash mismatch. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_target_length_mismatch_is_rejected() { + let path = "datadog/2/APM_TRACING/cfgid/config"; + let content: &[u8] = b"apm config payload"; + let (cfg, raw_delegated, dir) = matched_pair(path, content).await; + + let entries: &[(&str, &[u8])] = &[(path, content)]; + let mut resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, entries); + // Extend the bytes so the length no longer matches the director metadata. + let mut too_long = content.to_vec(); + too_long.extend_from_slice(b" extra trailing bytes"); + resp.target_files[0].raw = too_long; + + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("target with mismatching length must be rejected"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("bad length for file"), + "expected a length-mismatch error, got: {msg}" + ); + assert!(state.target_files_by_path.lock().unwrap().is_empty()); +} + +/// The director publishes a target whose path is not covered by any config +/// delegation. `verify_director_against_config` must reject the whole apply +/// before we ever hit the byte-level checks. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_director_target_not_in_config_delegations_is_rejected() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + // Config only delegates APM_TRACING... + let authorized_path = "datadog/2/APM_TRACING/cfgid/config"; + let authorized_content: &[u8] = b"apm payload"; + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + &[(authorized_path, authorized_content)], + &["datadog/*/APM_TRACING/*/*"], + &[HashAlgorithm::Sha256], + ) + .await; + + // ...but the director signs BOTH the authorized APM_TRACING target and an + // ASM target that no delegation ever authorized. + let unauthorized_path = "datadog/2/ASM/cfgid/config"; + let unauthorized_content: &[u8] = b"asm payload"; + let dir_entries: &[(&str, &[u8])] = &[ + (authorized_path, authorized_content), + (unauthorized_path, unauthorized_content), + ]; + let dir = + build_director_with_targets(&director_key, dir_entries, &[HashAlgorithm::Sha256]).await; + + let resp = delegated_response(&cfg, raw_delegated, "APM_TRACING", &dir, dir_entries); + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("unauthorized director target must be rejected"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("not authorized by config delegations"), + "expected an unauthorized-target error, got: {msg}" + ); + // The authorized sibling must NOT have been cached: apply() is all-or-nothing. + assert!(state.target_files_by_path.lock().unwrap().is_empty()); +} + +/// Director and config both authorize the same path, but their signed +/// metadata disagrees on the target's length (different content per builder, +/// with different sizes). `verify_director_against_config` compares length +/// first, so this is the error we expect — not a hash mismatch. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_director_config_length_disagreement_is_rejected() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let path = "datadog/2/APM_TRACING/cfgid/config"; + let config_content: &[u8] = b"short"; + let director_content: &[u8] = b"a distinctly longer payload"; + assert_ne!(config_content.len(), director_content.len()); + + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + &[(path, config_content)], + &["datadog/*/APM_TRACING/*/*"], + &[HashAlgorithm::Sha256], + ) + .await; + let dir = build_director_with_targets( + &director_key, + &[(path, director_content)], + &[HashAlgorithm::Sha256], + ) + .await; + + // The wire bytes match the director (that is what the fetcher would try + // to verify against downstream), but apply() must fail earlier during + // director/config cross-check. + let resp = delegated_response( + &cfg, + raw_delegated, + "APM_TRACING", + &dir, + &[(path, director_content)], + ); + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("director/config length disagreement must be rejected"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("length mismatch between director and config"), + "expected a director/config length-mismatch error, got: {msg}" + ); + assert!(state.target_files_by_path.lock().unwrap().is_empty()); +} + +/// Director and config agree on hash algorithms and content length but the +/// hash *values* differ (they signed different bytes of the same length). +/// The existing `test_director_hash_superset_is_rejected` covers the case +/// where the algorithm SET differs; this one exercises the equal-cardinality +/// value-disagreement path. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_director_config_hash_value_disagreement_is_rejected() { + let config_key = new_key(); + let product_key = new_key(); + let director_key = new_key(); + + let path = "datadog/2/APM_TRACING/cfgid/config"; + // Same length, different bytes: same-size hash sets but different values. + let config_content: &[u8] = b"config-side-payload"; // 19 bytes + let director_content: &[u8] = b"director-sd-payload"; // 19 bytes + assert_eq!(config_content.len(), director_content.len()); + assert_ne!(config_content, director_content); + + let (cfg, raw_delegated) = build_config_with_delegation( + &config_key, + &product_key, + "APM_TRACING", + &[(path, config_content)], + &["datadog/*/APM_TRACING/*/*"], + &[HashAlgorithm::Sha256], + ) + .await; + let dir = build_director_with_targets( + &director_key, + &[(path, director_content)], + &[HashAlgorithm::Sha256], + ) + .await; + + let resp = delegated_response( + &cfg, + raw_delegated, + "APM_TRACING", + &dir, + &[(path, director_content)], + ); + let http = MockHttp::new(); + http.push(&resp); + + let mut f = fetcher( + http.clone(), + cfg.root().unwrap().as_bytes().to_vec(), + dir.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("director/config hash-value disagreement must be rejected"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("hash set mismatch"), + "expected a director/config hash-value mismatch error, got: {msg}" + ); + assert!(state.target_files_by_path.lock().unwrap().is_empty()); +} + +// ---- org UUID tests ----------------------------------------------------- + +/// Build a config snapshot + timestamp pair whose signed snapshot carries +/// `custom.org_uuid = uuid`. Reuses the (version, expires, meta) already +/// signed inside `cfg.snapshot()`, adds the custom field, and re-signs with +/// `config_key`. The timestamp is rebuilt from the new snapshot so its +/// (length, hash) description stays consistent. +fn config_snapshot_with_org_uuid( + cfg: &RawSignedMetadataSet, + config_key: &Ed25519PrivateKey, + uuid: &str, +) -> (Vec, Vec) { + // Grab the `signed` object out of the raw snapshot bytes, inject + // `custom.org_uuid`, then re-hydrate a `SnapshotMetadata` via serde (the + // rust-tuf shim flattens unknown top-level fields into `additional_fields`, + // so our injected `custom` object lands in the right place). + let mut signed: serde_json::Value = + serde_json::from_slice(cfg.snapshot().unwrap().as_bytes()).unwrap(); + signed + .get_mut("signed") + .unwrap() + .as_object_mut() + .unwrap() + .insert( + "custom".to_string(), + serde_json::json!({ "org_uuid": uuid }), + ); + let new_snap: SnapshotMetadata = serde_json::from_value(signed["signed"].clone()).unwrap(); + let signed_snap = SignedMetadata::::new(&new_snap, config_key).unwrap(); + let raw_snap = signed_snap.to_raw().unwrap().as_bytes().to_vec(); + + let signed_ts = + TimestampMetadataBuilder::from_snapshot::(&signed_snap, &[HashAlgorithm::Sha256]) + .unwrap() + .signed::(config_key) + .unwrap(); + let raw_ts = signed_ts.to_raw().unwrap().as_bytes().to_vec(); + + (raw_snap, raw_ts) +} + +/// First-poll happy path: the concurrent org-data prefetch returns the same +/// UUID that the snapshot pins, so the fetch converges and exactly one org +/// request was issued. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_org_uuid_match_via_concurrent_prefetch() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let dir1 = build_v1(&director_key).await; + let (snap_bytes, ts_bytes) = config_snapshot_with_org_uuid(&cfg1, &config_key, "ORG-1"); + + let http = MockHttp::new(); + http.set_org_uuid("ORG-1"); + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + f.fetch_config(dummy_client(), &cache) + .await + .expect("matching org UUID must not fail the poll"); + + // Exactly one org request on the first poll (concurrent prefetch). + assert_eq!(http.org_request_count(), 1); +} + +/// Mismatching org UUID must fail the poll AND reset the fetcher, so a +/// subsequent poll reports the clean (embedded) versions. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_org_uuid_mismatch_fails_and_resets() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let dir1 = build_v1(&director_key).await; + let (snap_bytes, ts_bytes) = config_snapshot_with_org_uuid(&cfg1, &config_key, "ORG-EXPECTED"); + + let http = MockHttp::new(); + http.set_org_uuid("ORG-OTHER"); + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + let Err(err) = f.fetch_config(dummy_client(), &cache).await else { + panic!("mismatching org UUID must fail the poll"); + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("org UUID"), + "expected an org-UUID mismatch error, got: {msg}" + ); + + // Reset restored the pinned-root state. + assert_eq!(config_root_version(&f), 1); + assert_eq!(config_snapshot_version(&f), None); + // The one-shot prefetch flag was rearmed, so a fresh poll would prefetch again. + assert!(!f.org_data_prefetched); + assert!(f.org_uuid.is_none()); +} + +/// A config-root rotation invalidates the pinned UUID (it is keyed by the +/// config trusted-root version), forcing a fresh /api/v0.1/org fetch on the +/// next poll. +#[tokio::test] +#[cfg_attr(miri, ignore)] +async fn test_org_uuid_refetched_on_root_rotation() { + let config_key = new_key(); + let director_key = new_key(); + + let cfg1 = build_v1(&config_key).await; + let cfg2 = rotate_root(&config_key, &cfg1).await; // config root v1 -> v2 + let dir1 = build_v1(&director_key).await; + let (snap_bytes, ts_bytes) = config_snapshot_with_org_uuid(&cfg1, &config_key, "ORG-1"); + + let http = MockHttp::new(); + http.set_org_uuid("ORG-1"); + // Poll 1: config root v1, snapshot pins ORG-1. + http.push(&response( + &[cfg1.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + // Poll 2: config root rotates to v2; same signed snapshot/timestamp + // (rust-tuf purges non-root metadata on rotation, so the backend has to + // resend them). The pinned UUID binding keyed at root v1 no longer + // matches root v2 and forces a fresh /api/v0.1/org fetch. + http.push(&response( + &[cfg2.root().unwrap().as_bytes()], + &ts_bytes, + &snap_bytes, + Some(cfg1.targets().unwrap().as_bytes()), + vec![], + &dir1, + )); + + let mut f = fetcher( + http.clone(), + cfg1.root().unwrap().as_bytes().to_vec(), + dir1.root().unwrap().as_bytes().to_vec(), + ) + .await; + let state = test_state(); + let storage = NoopStorage; + let cache = TargetCache::new(&state, &storage); + + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 1); + assert_eq!(http.org_request_count(), 1); // concurrent prefetch + + f.fetch_config(dummy_client(), &cache).await.unwrap(); + assert_eq!(config_root_version(&f), 2); + // A second org fetch happened lazily because the config root rotated. + assert_eq!(http.org_request_count(), 2); +} diff --git a/libdd-remote-config/src/fetch/fetcher.rs b/libdd-remote-config/src/fetch/fetcher.rs index 30d7ccdcf8..5190dcb948 100644 --- a/libdd-remote-config/src/fetch/fetcher.rs +++ b/libdd-remote-config/src/fetch/fetcher.rs @@ -1,6 +1,9 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "agentless")] +use super::agentless; + use crate::targets::{Root, TargetsList}; use crate::{RemoteConfigCapabilities, RemoteConfigPath, RemoteConfigProduct, Target}; use base64::Engine; @@ -16,6 +19,7 @@ use libdd_trace_protobuf::remoteconfig::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256, Sha512}; use std::collections::HashSet; +use std::marker::PhantomData; use std::ops::Add; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; @@ -51,14 +55,26 @@ pub struct ConfigInvariants { pub language: String, pub tracer_version: String, pub endpoint: Endpoint, + #[cfg(feature = "agentless")] + /// Enables and configures agentless mode. If some the fetcher will + /// talk directly to the RC backend + pub agentless: Option, + #[cfg(not(feature = "agentless"))] + pub agentless: Option, +} + +impl ConfigInvariants { + pub fn agentless_enabled(&self) -> bool { + self.agentless.is_some() + } } -struct StoredTargetFile { - hash: String, - handle: Arc, - state: ConfigState, - meta: TargetFileMeta, - expiring: bool, +pub(crate) struct StoredTargetFile { + pub(crate) hash: String, + pub(crate) handle: Arc, + pub(crate) state: ConfigState, + pub(crate) meta: TargetFileMeta, + pub(crate) expiring: bool, } pub enum ConfigApplyState { @@ -105,7 +121,8 @@ impl ConfigProductCapabilities { } pub struct ConfigFetcherState { - target_files_by_path: Mutex, StoredTargetFile>>, + pub(in crate::fetch) target_files_by_path: + Mutex, StoredTargetFile>>, pub invariants: ConfigInvariants, endpoint: Endpoint, pub expire_unused_files: bool, @@ -155,10 +172,28 @@ impl ConfigFetcherFilesLock<'_, S> { impl ConfigFetcherState { pub fn with_client(invariants: ConfigInvariants, http_client: C) -> Self { + let (endpoint, agentless) = match &invariants.agentless { + Some(agentless_cfg) => { + #[cfg(feature = "agentless")] + { + ( + agentless_cfg.agentless_endpoint().clone(), + Some(agentless_cfg.clone()), + ) + } + + #[cfg(not(feature = "agentless"))] + match *agentless_cfg {} + } + None => (make_agent_configs_endpoint(&invariants.endpoint), None), + }; ConfigFetcherState { target_files_by_path: Default::default(), - endpoint: get_agent_configs_endpoint(&invariants.endpoint), - invariants, + endpoint, + invariants: ConfigInvariants { + agentless, + ..invariants + }, expire_unused_files: true, http_client, } @@ -203,9 +238,17 @@ impl ConfigFetcherState { } } +#[allow(clippy::large_enum_variant)] +enum FetcherMode { + Agent(PhantomData), + #[cfg(feature = "agentless")] + Agentless(agentless::AgentlessFetcher), +} + pub struct ConfigFetcher { pub file_storage: S, state: Arc>, + mode: FetcherMode, } pub struct ConfigClientState { @@ -217,6 +260,8 @@ pub struct ConfigClientState { /// Services discovered at runtime. Sent to the agent on each poll so it can route configs /// targeting those services to this client. Updated out-of-band by the consumer extra_services: Vec, + /// Server-recommended interval between consecutive polls. + refresh_interval: Option, } impl Default for ConfigClientState { @@ -228,6 +273,7 @@ impl Default for ConfigClientState { root_version: 1, last_error: None, extra_services: vec![], + refresh_interval: None, } } } @@ -236,14 +282,40 @@ impl ConfigClientState { pub fn set_extra_services(&mut self, services: Vec) { self.extra_services = services; } + + pub fn server_recommended_refresh_interval(&self) -> Option { + self.refresh_interval + } } impl ConfigFetcher { - pub fn new(file_storage: S, state: Arc>) -> Self { - ConfigFetcher { + /// Create a new config fetcher + /// This is guaranteed to be immediate (no await point) if `state.invariants.agentless_enabled` + /// is false + pub async fn new( + file_storage: S, + state: Arc>, + ) -> anyhow::Result { + #[cfg(feature = "agentless")] + let mode: FetcherMode = match &state.invariants.agentless { + Some(agentless_cfg) => FetcherMode::Agentless( + agentless::AgentlessFetcher::new( + agentless_cfg.clone(), + state.endpoint.clone(), + state.http_client.clone(), + ) + .await?, + ), + None => FetcherMode::Agent(PhantomData), + }; + #[cfg(not(feature = "agentless"))] + let mode: FetcherMode = FetcherMode::Agent(PhantomData); + + Ok(ConfigFetcher { file_storage, state, - } + mode, + }) } /// Sets the apply state on a stored file. @@ -326,34 +398,13 @@ impl ConfigFetcher anyhow::Result>>> { - let config_req = self.build_config_request( - runtime_id, - target, - product_capabilities, - client_id, - &*client_state, - ); - trace!("Submitting remote config request: {config_req:?}"); - + trace!("Submitting remote config request"); let req = self .state .endpoint @@ -520,14 +571,14 @@ impl ConfigFetcher ConfigFetcher ConfigFetcher anyhow::Result>>> { + let config_req = self.build_config_request( + runtime_id, + target, + product_capabilities, + client_id, + &*client_state, + ); + match &mut self.mode { + FetcherMode::Agent(_) => self.fetch_agent(config_req, target, client_state).await, + #[cfg(feature = "agentless")] + FetcherMode::Agentless(agentless_fetcher) => { + #[allow(clippy::expect_used)] + let client = config_req.client.expect( + "RC ConfigFetcher::build_config_request should always return a `Some` client", + ); + + let cache = agentless::TargetCache::new(&self.state, &self.file_storage); + let res = match agentless_fetcher.fetch_config(client, &cache).await { + Ok(r) => r, + Err(e) => { + client_state.last_error = Some(format!("{e:#}")); + // Surface the recommended backoff to the consumer of + // `ConfigClientState::server_recommended_refresh_interval` + // so it waits before the next attempt. `None` means + // "no extra backoff, use the regular interval". + if let Some(backoff) = agentless_fetcher.next_backoff() { + client_state.refresh_interval = Some(backoff); + } + return Err(e); + } + }; + + client_state.root_version = res.root_version; + client_state.targets_version = res.target_version; + client_state.refresh_interval = Some(res.refresh_interval); + if res.opaque_backend_state != client_state.opaque_backend_state { + client_state.opaque_backend_state = res.opaque_backend_state.clone(); + } + client_state.last_error = None; + + let config_paths: HashSet = res + .targets + .iter() + .map(|target_ref| target_ref.path.clone()) + .collect(); + + let configs = cache.lock().collect_handles(&res.targets)?; + + client_state.last_config_paths = config_paths; + Ok(Some(configs)) + } + } + } } -fn get_agent_configs_endpoint(endpoint: &Endpoint) -> Endpoint { +fn make_agent_configs_endpoint(endpoint: &Endpoint) -> Endpoint { let mut parts = endpoint.url.clone().into_parts(); parts.path_and_query = Some(PathAndQuery::from_static("/v0.7/config")); #[allow(clippy::unwrap_used)] @@ -602,27 +726,21 @@ fn get_agent_configs_endpoint(endpoint: &Endpoint) -> Endpoint { pub mod tests { use super::*; use crate::fetch::test_server::RemoteConfigServer; - use crate::RemoteConfigSource; use http::Response; use libdd_capabilities_impl::NativeCapabilities; use libdd_common::http_common; use std::mem::transmute; use std::sync::LazyLock; - pub(crate) static PATH_FIRST: LazyLock = LazyLock::new(|| RemoteConfigPath { - source: RemoteConfigSource::Employee, - product: RemoteConfigProduct::ApmTracing, - config_id: "1234".to_string(), - name: "config".to_string(), + pub(crate) static PATH_FIRST: LazyLock = LazyLock::new(|| { + RemoteConfigPath::parse("employee/APM_TRACING/1234/config") + .expect("PATH_FIRST is a valid remote config path") }); - pub(crate) static PATH_SECOND: LazyLock = - LazyLock::new(|| RemoteConfigPath { - source: RemoteConfigSource::Employee, - product: RemoteConfigProduct::ApmTracing, - config_id: "9876".to_string(), - name: "config".to_string(), - }); + pub(crate) static PATH_SECOND: LazyLock = LazyLock::new(|| { + RemoteConfigPath::parse("employee/APM_TRACING/9876/config") + .expect("PATH_SECOND is a valid remote config path") + }); pub(crate) static DUMMY_TARGET: LazyLock> = LazyLock::new(|| { Arc::new(Target::new( @@ -722,7 +840,9 @@ pub mod tests { server.dummy_options().invariants, NativeCapabilities::new_without_connection_pooling(), )), - ); + ) + .await + .unwrap(); let mut opaque_state = ConfigClientState::default(); let mut response = http_common::empty_response(Response::builder()).unwrap(); @@ -760,7 +880,9 @@ pub mod tests { server.dummy_options().invariants, NativeCapabilities::new_without_connection_pooling(), )), - ); + ) + .await + .unwrap(); let mut opaque_state = ConfigClientState::default(); let capabilities = server.dummy_product_capabilities(); @@ -862,6 +984,7 @@ pub mod tests { language: "php".to_string(), tracer_version: "1.2.3".to_string(), endpoint: server.endpoint.clone(), + agentless: None, }; let product_capabilities = ConfigProductCapabilities::new( vec![ @@ -877,7 +1000,10 @@ pub mod tests { invariants, NativeCapabilities::new_without_connection_pooling(), )), - ); + ) + .await + .unwrap(); + let mut opaque_state = ConfigClientState::default(); { @@ -1064,7 +1190,10 @@ pub mod tests { server.dummy_options().invariants, NativeCapabilities::new_without_connection_pooling(), )), - ); + ) + .await + .unwrap(); + let mut opaque_state = ConfigClientState::default(); // Default: nothing set, agent receives an empty list. @@ -1165,7 +1294,10 @@ pub mod tests { server.dummy_options().invariants, NativeCapabilities::new_without_connection_pooling(), )), - ); + ) + .await + .unwrap(); + let mut opaque_state = ConfigClientState::default(); let fetched = fetcher diff --git a/libdd-remote-config/src/fetch/mod.rs b/libdd-remote-config/src/fetch/mod.rs index 44e048e4ef..887a1a5f71 100644 --- a/libdd-remote-config/src/fetch/mod.rs +++ b/libdd-remote-config/src/fetch/mod.rs @@ -1,15 +1,22 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +#[cfg(feature = "agentless")] +mod agentless; + mod fetcher; #[cfg(not(target_arch = "wasm32"))] mod multitarget; #[cfg(not(target_arch = "wasm32"))] mod shared; mod single; + #[cfg(any(test, feature = "test"))] pub mod test_server; +#[cfg(feature = "agentless")] +pub use agentless::*; + #[allow(clippy::useless_attribute)] // different clippy versions are differently picky #[cfg_attr(test, allow(ambiguous_glob_reexports))] // ignore mod tests re-export pub use fetcher::*; diff --git a/libdd-remote-config/src/fetch/shared.rs b/libdd-remote-config/src/fetch/shared.rs index 4313bb834f..ffd7c0d63a 100644 --- a/libdd-remote-config/src/fetch/shared.rs +++ b/libdd-remote-config/src/fetch/shared.rs @@ -278,7 +278,13 @@ impl SharedFetcher { S::StoredFile: RefcountedFile, { let state = storage.state.clone(); - let mut fetcher = ConfigFetcher::new(storage, state); + let mut fetcher = match ConfigFetcher::new(storage, state).await { + Ok(f) => f, + Err(e) => { + error!("failed to create the fetcher: {:?}", e); + return; + } + }; let mut opaque_state = ConfigClientState::default(); @@ -317,7 +323,9 @@ impl SharedFetcher { }; match fetched { - Ok(None) => clean_inactive(), // nothing changed + Ok(None) => { + clean_inactive(); + } Ok(Some(files)) => { if !files.is_empty() || !last_files.is_empty() { for file in files.iter() { @@ -352,6 +360,17 @@ impl SharedFetcher { } } + if let Some(interval) = opaque_state.server_recommended_refresh_interval() { + // Keep the run-loop interval in sync with the server-provided value. + // If the interval in nanoseconds is greater than u64::MAX, pick the max + // representable value. This is tolerable as u64::MAX nanoseconds + // still represents 35584 years. + self.interval.store( + interval.as_nanos().min(u64::MAX as u128) as u64, + Ordering::Relaxed, + ); + } + select! { _ = self.cancellation.cancelled() => { break; } _ = sleep(Duration::from_nanos(self.interval.load(Ordering::Relaxed))) => {} diff --git a/libdd-remote-config/src/fetch/single.rs b/libdd-remote-config/src/fetch/single.rs index 1b1ecdf054..633ba27d8d 100644 --- a/libdd-remote-config/src/fetch/single.rs +++ b/libdd-remote-config/src/fetch/single.rs @@ -9,6 +9,9 @@ use crate::file_change_tracker::{Change, ChangeTracker, FilePath, UpdatedFiles}; use crate::{RemoteConfigCapabilities, RemoteConfigPath, RemoteConfigProduct, Target}; use libdd_capabilities::{HttpClientCapability, SleepCapability}; use std::sync::Arc; +use std::time::Duration; + +const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(5); /// Simple implementation pub struct SingleFetcher { @@ -17,7 +20,7 @@ pub struct SingleFetcher SingleFetcher { - pub fn new( + pub async fn new( sink: S, target: Target, runtime_id: String, options: ConfigOptions, http_client: C, - ) -> Self { - SingleFetcher { + ) -> anyhow::Result { + Ok(SingleFetcher { fetcher: ConfigFetcher::new( sink, Arc::new(ConfigFetcherState::with_client( options.invariants, http_client, )), + ) + .await?, + target: Arc::new(target), + product_capabilities: ConfigProductCapabilities::new( + options.products, + options.capabilities, ), + runtime_id, + client_id: uuid::Uuid::new_v4().to_string(), + client_state: ConfigClientState::default(), + }) + } + + pub fn new_no_agentless( + sink: S, + target: Target, + runtime_id: String, + options: ConfigOptions, + http_client: C, + ) -> anyhow::Result { + Ok(SingleFetcher { + fetcher: futures::executor::block_on(ConfigFetcher::new( + sink, + Arc::new(ConfigFetcherState::with_client( + options.invariants, + http_client, + )), + ))?, target: Arc::new(target), product_capabilities: ConfigProductCapabilities::new( options.products, @@ -50,8 +80,8 @@ impl SingleFetcher Self { @@ -67,15 +97,24 @@ impl SingleFetcher &String { + pub fn get_client_id(&self) -> &str { &self.client_id } + /// Returns the server-recommended interval before the next poll. + /// In agentless mode this is updated after every successful fetch. + /// In agent mode it returns the default of 5 seconds. + pub fn get_refresh_interval(&self) -> Duration { + self.client_state + .server_recommended_refresh_interval() + .unwrap_or(DEFAULT_REFRESH_INTERVAL) + } + /// Accesses the underlying file storage (the [`ConfigFetcher`]'s `file_storage`). pub fn file_storage(&self) -> &S { &self.fetcher.file_storage @@ -90,7 +129,7 @@ impl SingleFetcher) { - self.opaque_state.set_extra_services(services); + self.client_state.set_extra_services(services); } /// Replace the set of subscribed products and capabilities. @@ -118,17 +157,36 @@ impl SingleChangesFet where S::StoredFile: FilePath, { - pub fn new( + pub async fn new( + sink: S, + target: Target, + runtime_id: String, + options: ConfigOptions, + http_client: C, + ) -> anyhow::Result { + Ok(SingleChangesFetcher { + changes: ChangeTracker::default(), + fetcher: SingleFetcher::new(sink, target, runtime_id, options, http_client).await?, + }) + } + + pub fn new_no_agentless( sink: S, target: Target, runtime_id: String, options: ConfigOptions, http_client: C, - ) -> Self { - SingleChangesFetcher { + ) -> anyhow::Result { + Ok(SingleChangesFetcher { changes: ChangeTracker::default(), - fetcher: SingleFetcher::new(sink, target, runtime_id, options, http_client), - } + fetcher: SingleFetcher::new_no_agentless( + sink, + target, + runtime_id, + options, + http_client, + )?, + }) } pub fn with_client_id(mut self, client_id: String) -> Self { @@ -149,10 +207,16 @@ where }) } - pub fn get_client_id(&self) -> &String { + pub fn get_client_id(&self) -> &str { self.fetcher.get_client_id() } + /// Returns the interval before the next poll. + /// See [`SingleFetcher::get_refresh_interval`]. + pub fn get_refresh_interval(&self) -> Duration { + self.fetcher.get_refresh_interval() + } + /// Sets the apply state on a stored file. pub fn set_config_state(&self, file: &S::StoredFile, state: ConfigApplyState) { self.fetcher.set_config_state(file.path(), state) diff --git a/libdd-remote-config/src/fetch/test_server.rs b/libdd-remote-config/src/fetch/test_server.rs index 1ba5eef9ac..e51b92b9cd 100644 --- a/libdd-remote-config/src/fetch/test_server.rs +++ b/libdd-remote-config/src/fetch/test_server.rs @@ -93,8 +93,8 @@ impl RemoteConfigServer { && applied_files.len() == states.len() && states.iter().all(|s| { for (p, (_, v, _)) in applied_files.iter() { - if p.product.to_string() == s.product - && p.config_id == s.id + if p.product().to_string() == s.product + && p.config_id() == s.id && *v == s.version { return true; @@ -226,6 +226,7 @@ impl RemoteConfigServer { language: "php".to_string(), tracer_version: "1.2.3".to_string(), endpoint: self.endpoint.clone(), + agentless: None, }, products: vec![ RemoteConfigProduct::ApmTracing, diff --git a/libdd-remote-config/src/file_change_tracker.rs b/libdd-remote-config/src/file_change_tracker.rs index 16b2323b58..fbdfa7491c 100644 --- a/libdd-remote-config/src/file_change_tracker.rs +++ b/libdd-remote-config/src/file_change_tracker.rs @@ -82,7 +82,7 @@ impl ChangeTracker { #[cfg(test)] mod tests { use super::*; - use crate::{RemoteConfigPath, RemoteConfigProduct, RemoteConfigSource}; + use crate::RemoteConfigPath; struct TestFile(RemoteConfigPath); @@ -93,12 +93,10 @@ mod tests { } fn file(name: &str) -> Arc { - Arc::new(TestFile(RemoteConfigPath { - source: RemoteConfigSource::Employee, - product: RemoteConfigProduct::ApmTracing, - config_id: "id".to_string(), - name: name.to_string(), - })) + Arc::new(TestFile( + RemoteConfigPath::parse(&format!("employee/APM_TRACING/id/{name}")) + .expect("test file has a valid path"), + )) } #[test] @@ -115,11 +113,11 @@ mod tests { assert_eq!(changes.len(), 2); match &changes[0] { - Change::Remove(f) => assert_eq!(f.path().name, "a"), + Change::Remove(f) => assert_eq!(f.path().name(), "a"), _ => panic!("expected the removal to be emitted first"), } match &changes[1] { - Change::Add(f) => assert_eq!(f.path().name, "b"), + Change::Add(f) => assert_eq!(f.path().name(), "b"), _ => panic!("expected the addition to be emitted after the removal"), } } diff --git a/libdd-remote-config/src/file_storage.rs b/libdd-remote-config/src/file_storage.rs index 612dfdc644..f70f3b7cc5 100644 --- a/libdd-remote-config/src/file_storage.rs +++ b/libdd-remote-config/src/file_storage.rs @@ -144,7 +144,7 @@ impl ParseFile for RegistryParser { type Parsed = anyhow::Result>; fn parse(&self, path: &RemoteConfigPath, contents: Vec) -> Self::Parsed { - self.0.parse(path.product, &contents) + self.0.parse(path.product(), &contents) } } diff --git a/libdd-remote-config/src/parse.rs b/libdd-remote-config/src/parse.rs index f890e2cb29..cab91b9116 100644 --- a/libdd-remote-config/src/parse.rs +++ b/libdd-remote-config/src/parse.rs @@ -7,7 +7,7 @@ use crate::{ agent_task::{self, AgentTaskFile}, dynamic::{self, DynamicConfigFile}, }, - RemoteConfigPath, RemoteConfigProduct, RemoteConfigSource, + RemoteConfigPath, RemoteConfigProduct, }; use std::collections::HashMap; use std::fmt::{Debug, Display, Formatter, Result}; @@ -185,35 +185,26 @@ pub fn default_registry() -> ParserRegistry { // ── RemoteConfigValue ───────────────────────────────────────────────────────── pub struct RemoteConfigValue { - pub source: RemoteConfigSource, - pub product: RemoteConfigProduct, + pub path: RemoteConfigPath, pub data: Option, - pub config_id: String, - pub name: String, } impl Debug for RemoteConfigValue { fn fmt(&self, f: &mut Formatter<'_>) -> Result { f.debug_struct("RemoteConfigValue") - .field("source", &self.source) - .field("product", &self.product) - .field("config_id", &self.config_id) - .field("name", &self.name) + .field("source", &self.path.source()) + .field("product", &self.path.product()) + .field("config_id", &self.path.config_id()) + .field("name", &self.path.name()) .finish() } } impl RemoteConfigValue { pub fn try_parse(path: &str, data: &[u8], registry: &ParserRegistry) -> anyhow::Result { - let path = RemoteConfigPath::try_parse(path)?; - let data = registry.parse(path.product, data)?; - Ok(RemoteConfigValue { - source: path.source, - product: path.product, - data, - config_id: path.config_id.to_string(), - name: path.name.to_string(), - }) + let path: RemoteConfigPath = RemoteConfigPath::try_parse(path)?.into(); + let data = registry.parse(path.product(), data)?; + Ok(RemoteConfigValue { data, path }) } } diff --git a/libdd-remote-config/src/path.rs b/libdd-remote-config/src/path.rs index 649644245e..1cc98980ae 100644 --- a/libdd-remote-config/src/path.rs +++ b/libdd-remote-config/src/path.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; -use std::hash::Hash; +use std::hash::{Hash, Hasher}; use std::str::FromStr; use std::sync::Arc; @@ -42,81 +42,218 @@ pub enum RemoteConfigProduct { LiveDebuggingSymbolDb, } -#[derive(Debug, Clone, Eq, Hash, PartialEq)] +#[derive(Clone)] pub struct RemoteConfigPath { - pub source: RemoteConfigSource, - pub product: RemoteConfigProduct, - pub config_id: String, - pub name: String, + raw: Box, + source: RemoteConfigSource, + product: RemoteConfigProduct, + /// Byte offset in `raw` where the `config_id` segment starts. + /// The segment ends at `name_start - 1` (the `/` before `name`). + config_id_start: u32, + /// Byte offset in `raw` where the `name` segment starts. It runs to the + /// end of `raw`. + name_start: u32, } -#[derive(Debug, Clone, Eq, Hash, PartialEq)] +impl std::fmt::Debug for RemoteConfigPath { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RemoteConfigPath") + .field("source", &self.source) + .field("product", &self.product) + .field("config_id", &self.config_id()) + .field("name", &self.name()) + .finish() + } +} + +impl PartialEq for RemoteConfigPath { + fn eq(&self, other: &Self) -> bool { + self.raw == other.raw + } +} +impl Eq for RemoteConfigPath {} +impl Hash for RemoteConfigPath { + fn hash(&self, state: &mut H) { + self.raw.hash(state); + } +} + +#[derive(Debug, Copy, Clone)] pub struct RemoteConfigPathRef<'a> { - pub source: RemoteConfigSource, - pub product: RemoteConfigProduct, - pub config_id: &'a str, - pub name: &'a str, + raw: &'a str, + source: RemoteConfigSource, + product: RemoteConfigProduct, + config_id_start: u32, + name_start: u32, +} + +impl PartialEq for RemoteConfigPathRef<'_> { + fn eq(&self, other: &Self) -> bool { + self.raw == other.raw + } +} +impl Eq for RemoteConfigPathRef<'_> {} +impl Hash for RemoteConfigPathRef<'_> { + fn hash(&self, state: &mut H) { + self.raw.hash(state); + } } impl RemoteConfigPath { pub fn try_parse(path: &str) -> anyhow::Result> { - let parts: Vec<_> = path.split('/').collect(); - Ok(RemoteConfigPathRef { - source: match parts[0] { - "datadog" => { - if parts.len() != 5 { - anyhow::bail!("{} is datadog and does not have exactly 5 parts", path); - } - RemoteConfigSource::Datadog(parts[1].parse()?) - } - "employee" => { - if parts.len() != 4 { - anyhow::bail!("{} is employee and does not have exactly 4 parts", path); - } - RemoteConfigSource::Employee - } - source => anyhow::bail!("Unknown source {}", source), - }, - product: RemoteConfigProduct::from_str(parts[parts.len() - 3]) - .map_err(|_| anyhow::anyhow!("Unknown product {}", parts[parts.len() - 3]))?, - config_id: parts[parts.len() - 2], - name: parts[parts.len() - 1], + parse_into_ref(path) + } + + pub fn parse(path: &str) -> anyhow::Result { + let r = parse_into_ref(path)?; + Ok(Self { + raw: Box::from(r.raw), + source: r.source, + product: r.product, + config_id_start: r.config_id_start, + name_start: r.name_start, }) } + + #[inline] + pub fn as_str(&self) -> &str { + &self.raw + } + + #[inline] + pub fn source(&self) -> RemoteConfigSource { + self.source + } + + #[inline] + pub fn product(&self) -> RemoteConfigProduct { + self.product + } + + #[inline] + pub fn config_id(&self) -> &str { + &self.raw[self.config_id_start as usize..self.name_start as usize - 1] + } + + #[inline] + pub fn name(&self) -> &str { + &self.raw[self.name_start as usize..] + } } -impl Display for RemoteConfigPathRef<'_> { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self.source { - RemoteConfigSource::Datadog(id) => write!( - f, - "datadog/{}/{}/{}/{}", - id, self.product, self.config_id, self.name - ), - RemoteConfigSource::Employee => { - write!( - f, - "employee/{}/{}/{}", - self.product, self.config_id, self.name - ) +impl<'a> RemoteConfigPathRef<'a> { + #[inline] + pub fn as_str(&self) -> &'a str { + self.raw + } + + #[inline] + pub fn source(&self) -> RemoteConfigSource { + self.source + } + + #[inline] + pub fn product(&self) -> RemoteConfigProduct { + self.product + } + + #[inline] + pub fn config_id(&self) -> &'a str { + &self.raw[self.config_id_start as usize..self.name_start as usize - 1] + } + + #[inline] + pub fn name(&self) -> &'a str { + &self.raw[self.name_start as usize..] + } +} + +fn parse_into_ref(path: &str) -> anyhow::Result> { + let slash_positions: Vec = path.match_indices('/').map(|(i, _)| i).collect(); + let n_slashes = slash_positions.len(); + + // Every valid path has at least: source '/' ... '/' config_id '/' name. + // Datadog paths have 4 slashes (5 segments); employee paths have 3. + let first_slash = *slash_positions.first().ok_or_else(|| { + anyhow::format_err!("path {path} does not contain a '/', cannot be a remote config path") + })?; + + let source = match &path[..first_slash] { + "datadog" => { + if n_slashes != 4 { + anyhow::bail!("{path} is datadog and does not have exactly 5 parts"); } + let org_id_end = slash_positions[1]; + let org_id: u64 = path[first_slash + 1..org_id_end].parse()?; + // The agent parses org_id as an int64; reject values it would reject + // (> i64::MAX) so both clients accept/reject the same paths. + if org_id > i64::MAX as u64 { + anyhow::bail!("org_id {org_id} exceeds i64::MAX in path {path}"); + } + RemoteConfigSource::Datadog(org_id) } + "employee" => { + if n_slashes != 3 { + anyhow::bail!("{path} is employee and does not have exactly 4 parts"); + } + RemoteConfigSource::Employee + } + source => anyhow::bail!("Unknown source {source}"), + }; + + // Segments are indexed from the tail so both wire forms share this code. + // `slash_positions[n_slashes - 3]` = '/' before product. + // `slash_positions[n_slashes - 2]` = '/' before config_id. + // `slash_positions[n_slashes - 1]` = '/' before name. + let product_start = slash_positions[n_slashes - 3] + 1; + let product_end = slash_positions[n_slashes - 2]; + let config_id_start = slash_positions[n_slashes - 2] + 1; + let name_start = slash_positions[n_slashes - 1] + 1; + + let product = RemoteConfigProduct::from_str(&path[product_start..product_end]) + .map_err(|_| anyhow::format_err!("Unknown product {}", &path[product_start..product_end]))?; + + if name_start == config_id_start + 1 { + anyhow::bail!("empty config_id in path {path}"); + } + if name_start >= path.len() { + anyhow::bail!("empty name in path {path}"); + } + + let config_id_start_u32 = u32::try_from(config_id_start) + .map_err(|_| anyhow::format_err!("path {path} is too long (>= 4 GiB)"))?; + let name_start_u32 = u32::try_from(name_start) + .map_err(|_| anyhow::format_err!("path {path} is too long (>= 4 GiB)"))?; + + Ok(RemoteConfigPathRef { + raw: path, + source, + product, + config_id_start: config_id_start_u32, + name_start: name_start_u32, + }) +} + +impl Display for RemoteConfigPathRef<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.raw) } } impl Display for RemoteConfigPath { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - RemoteConfigPathRef::from(self).fmt(f) + f.write_str(&self.raw) } } impl<'a> From<&RemoteConfigPathRef<'a>> for RemoteConfigPath { fn from(from: &RemoteConfigPathRef<'a>) -> RemoteConfigPath { RemoteConfigPath { + raw: Box::from(from.raw), source: from.source, product: from.product, - config_id: from.config_id.to_owned(), - name: from.name.to_owned(), + config_id_start: from.config_id_start, + name_start: from.name_start, } } } @@ -129,25 +266,83 @@ impl<'a> From> for RemoteConfigPath { impl<'a> From<&'a RemoteConfigPath> for RemoteConfigPathRef<'a> { fn from(from: &'a RemoteConfigPath) -> RemoteConfigPathRef<'a> { RemoteConfigPathRef { + raw: &from.raw, source: from.source, product: from.product, - config_id: from.config_id.as_str(), - name: from.name.as_str(), + config_id_start: from.config_id_start, + name_start: from.name_start, } } } impl<'a> hashbrown::Equivalent> for RemoteConfigPathRef<'a> { fn equivalent(&self, key: &Arc) -> bool { - let RemoteConfigPathRef { - source, - product, - config_id, - name, - } = self; - source == &key.source - && product == &key.product - && config_id == &key.config_id - && name == &key.name + self.raw == key.raw.as_ref() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_datadog_roundtrip() { + let raw = "datadog/42/APM_TRACING/cfg-1/tracing.config"; + let p = RemoteConfigPath::parse(raw).unwrap(); + assert_eq!(p.as_str(), raw); + assert_eq!(p.source(), RemoteConfigSource::Datadog(42)); + assert_eq!(p.product(), RemoteConfigProduct::ApmTracing); + assert_eq!(p.config_id(), "cfg-1"); + assert_eq!(p.name(), "tracing.config"); + assert_eq!(p.to_string(), raw); + } + + #[test] + fn parse_employee_roundtrip() { + let raw = "employee/ASM_DD/blocklist/rules"; + let p = RemoteConfigPath::parse(raw).unwrap(); + assert_eq!(p.as_str(), raw); + assert_eq!(p.source(), RemoteConfigSource::Employee); + assert_eq!(p.product(), RemoteConfigProduct::AsmDd); + assert_eq!(p.config_id(), "blocklist"); + assert_eq!(p.name(), "rules"); + assert_eq!(p.to_string(), raw); + } + + #[test] + fn ref_matches_owned() { + let raw = "datadog/1/ASM/cfg/name"; + let r = RemoteConfigPath::try_parse(raw).unwrap(); + let o: RemoteConfigPath = r.into(); + assert_eq!(o.as_str(), raw); + assert_eq!(o.config_id(), "cfg"); + assert_eq!(o.name(), "name"); + } + + #[test] + fn hash_and_eq_by_wire_form() { + let a = RemoteConfigPath::parse("datadog/1/ASM/cfg/name").unwrap(); + let b = RemoteConfigPath::parse("datadog/1/ASM/cfg/name").unwrap(); + assert_eq!(a, b); + use std::collections::HashSet; + let mut s = HashSet::new(); + s.insert(a); + assert!(s.contains(&b)); + } + + #[test] + fn rejects_bad_paths() { + assert!(RemoteConfigPath::parse("").is_err()); + assert!(RemoteConfigPath::parse("nosource/x/y/z").is_err()); + assert!(RemoteConfigPath::parse("datadog/1/APM_TRACING/cfg").is_err()); + assert!(RemoteConfigPath::parse("datadog/1/APM_TRACING//name").is_err()); + assert!(RemoteConfigPath::parse("datadog/1/APM_TRACING/cfg/").is_err()); + assert!(RemoteConfigPath::parse("datadog/notanint/APM_TRACING/cfg/name").is_err()); + // org_id > i64::MAX must be rejected to stay in lockstep with the + // agent, which parses org_id as int64. + assert!( + RemoteConfigPath::parse("datadog/9223372036854775808/APM_TRACING/cfg/name").is_err() + ); + assert!(RemoteConfigPath::parse("employee/UNKNOWN_PRODUCT/cfg/name").is_err()); } } diff --git a/libdd-tracer-flare/Cargo.toml b/libdd-tracer-flare/Cargo.toml index 353731ef11..c0d5037504 100644 --- a/libdd-tracer-flare/Cargo.toml +++ b/libdd-tracer-flare/Cargo.toml @@ -37,4 +37,5 @@ tokio = { version = "1.36.0", features = ["time", "macros", "rt"] } [features] default = ["listener"] +agentless = ["libdd-remote-config/agentless"] listener = ["libdd-remote-config/client"] diff --git a/libdd-tracer-flare/src/lib.rs b/libdd-tracer-flare/src/lib.rs index 67b22e59b3..5798310123 100644 --- a/libdd-tracer-flare/src/lib.rs +++ b/libdd-tracer-flare/src/lib.rs @@ -182,6 +182,7 @@ impl TracerFlareManager { language, tracer_version, endpoint: remote_config_endpoint, + agentless: None, }, products: vec![ RemoteConfigProduct::AgentConfig, @@ -190,13 +191,16 @@ impl TracerFlareManager { capabilities: vec![], }; - tracer_flare.listener = Some(SingleChangesFetcher::new( - ParsedFileStorage::default(), - Target::new(service, env, app_version, vec![], vec![]), - runtime_id, - config_to_fetch, - NativeCapabilities::new_without_connection_pooling(), - )); + tracer_flare.listener = Some( + SingleChangesFetcher::new_no_agentless( + ParsedFileStorage::default(), + Target::new(service, env, app_version, vec![], vec![]), + runtime_id, + config_to_fetch, + NativeCapabilities::new_without_connection_pooling(), + ) + .map_err(|e| FlareError::ListeningError(e.to_string()))?, + ); Ok(tracer_flare) } @@ -523,7 +527,7 @@ mod tests { }, fetch::FileStorage, file_storage::ParsedFileStorage, - RemoteConfigPath, RemoteConfigProduct, RemoteConfigSource, + RemoteConfigPath, }; #[cfg(feature = "listener")] use std::sync::{atomic::Ordering, Arc}; @@ -631,12 +635,10 @@ mod tests { #[cfg(feature = "listener")] fn test_remote_config_with_valid_log_level() { let storage = ParsedFileStorage::default(); - let path = Arc::new(RemoteConfigPath { - product: RemoteConfigProduct::AgentConfig, - config_id: "test".to_string(), - name: "flare-log-level.test".to_string(), - source: RemoteConfigSource::Datadog(1), - }); + let path = Arc::new( + RemoteConfigPath::parse("datadog/1/AGENT_CONFIG/test/flare-log-level.test") + .expect("valid path"), + ); let config = AgentConfigFile { name: "flare-log-level.test".to_string(), @@ -657,12 +659,9 @@ mod tests { #[cfg(feature = "listener")] fn test_remote_config_with_send_task() { let storage = ParsedFileStorage::default(); - let path = Arc::new(RemoteConfigPath { - product: RemoteConfigProduct::AgentTask, - config_id: "test".to_string(), - name: "tracer_flare".to_string(), - source: RemoteConfigSource::Datadog(1), - }); + let path = Arc::new( + RemoteConfigPath::parse("datadog/1/AGENT_TASK/test/tracer_flare").expect("valid path"), + ); let task = AgentTaskFile { args: AgentTask { @@ -686,12 +685,10 @@ mod tests { #[cfg(feature = "listener")] fn test_remote_config_with_invalid_config() { let storage = ParsedFileStorage::default(); - let path = Arc::new(RemoteConfigPath { - product: RemoteConfigProduct::AgentConfig, - config_id: "test".to_string(), - name: "invalid-config".to_string(), - source: RemoteConfigSource::Datadog(1), - }); + let path = Arc::new( + RemoteConfigPath::parse("datadog/1/AGENT_CONFIG/test/invalid-config") + .expect("valid path"), + ); let config = AgentConfigFile { name: "invalid-config".to_string(), @@ -756,12 +753,10 @@ mod tests { let agent_config_file = storage .store( 1, - Arc::new(RemoteConfigPath { - product: RemoteConfigProduct::AgentConfig, - config_id: "test".to_string(), - name: "flare-log-level.test".to_string(), - source: RemoteConfigSource::Datadog(1), - }), + Arc::new( + RemoteConfigPath::parse("datadog/1/AGENT_CONFIG/test/flare-log-level.test") + .expect("valid path"), + ), serde_json::to_vec(&AgentConfigFile { name: "flare-log-level.test".to_string(), config: AgentConfig { @@ -791,12 +786,10 @@ mod tests { let error_file = storage .store( 2, - Arc::new(RemoteConfigPath { - product: RemoteConfigProduct::AgentConfig, - config_id: "error".to_string(), - name: "error".to_string(), - source: RemoteConfigSource::Datadog(1), - }), + Arc::new( + RemoteConfigPath::parse("datadog/1/AGENT_CONFIG/error/error") + .expect("valid path"), + ), b"invalid".to_vec(), ) .unwrap(); @@ -809,12 +802,10 @@ mod tests { #[cfg(feature = "listener")] fn test_check_remote_config_file_with_parsing_error() { let storage = ParsedFileStorage::default(); - let path = Arc::new(RemoteConfigPath { - product: RemoteConfigProduct::AgentConfig, - config_id: "test".to_string(), - name: "invalid-json".to_string(), - source: RemoteConfigSource::Datadog(1), - }); + let path = Arc::new( + RemoteConfigPath::parse("datadog/1/AGENT_CONFIG/test/invalid-json") + .expect("valid path"), + ); let file = storage .store(1, path.clone(), b"invalid json".to_vec())