diff --git a/Cargo.lock b/Cargo.lock index d223a679..ed9678e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1002,6 +1002,7 @@ dependencies = [ "serde_yaml", "static-analysis-kernel", "static-analysis-server", + "tempfile", "terminal-emoji", "thiserror 2.0.18", "tracing", diff --git a/crates/bins/Cargo.toml b/crates/bins/Cargo.toml index 86091cc8..3fa291d0 100644 --- a/crates/bins/Cargo.toml +++ b/crates/bins/Cargo.toml @@ -18,6 +18,9 @@ name = "datadog-export-rulesets" [[bin]] name = "datadog-static-analyzer-server" +[dev-dependencies] +tempfile = "3.23.0" + [dev-dependencies.cargo-husky] version = "1" default-features = false # Disable features which are enabled by default diff --git a/crates/bins/src/bin/datadog-static-analyzer-git-hook.rs b/crates/bins/src/bin/datadog-static-analyzer-git-hook.rs index 7c70d692..97d3964f 100644 --- a/crates/bins/src/bin/datadog-static-analyzer-git-hook.rs +++ b/crates/bins/src/bin/datadog-static-analyzer-git-hook.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result}; use cli::config_file::get_config; use cli::constants::{ - DEFAULT_MAX_CPUS, DEFAULT_MAX_FILE_SIZE_KB, EXIT_CODE_GITHOOK_FAILED, + DEFAULT_MAX_CPUS, DEFAULT_MAX_FILE_SIZE_KB, DEFAULT_TOOL_NAME, EXIT_CODE_GITHOOK_FAILED, EXIT_CODE_INVALID_CONFIGURATION, EXIT_CODE_INVALID_DIRECTORY, EXIT_CODE_NO_DIRECTORY, EXIT_CODE_NO_SECRET_OR_STATIC_ANALYSIS, EXIT_CODE_RULESET_NOT_FOUND, EXIT_CODE_RULE_CHECKSUM_INVALID, EXIT_CODE_SHA_OR_DEFAULT_BRANCH, @@ -626,12 +626,15 @@ fn main() -> Result<()> { &configuration, all_rule_results, secrets_results, + Vec::new(), SarifReportMetadata { add_git_info: false, debug: configuration.use_debug, config_digest: configuration.generate_diff_aware_digest(), diff_aware_parameters: None, execution_time_secs: analysis_start_instant.elapsed().as_secs(), + tool_name: DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &all_path_metadata, ) diff --git a/crates/bins/src/bin/datadog-static-analyzer.rs b/crates/bins/src/bin/datadog-static-analyzer.rs index 362b0e74..e541f83d 100644 --- a/crates/bins/src/bin/datadog-static-analyzer.rs +++ b/crates/bins/src/bin/datadog-static-analyzer.rs @@ -5,10 +5,10 @@ use std::path::PathBuf; use cli::config_file::get_config; use cli::constants::{ - DEFAULT_MAX_CPUS, DEFAULT_MAX_FILE_SIZE_KB, EXIT_CODE_FAIL_ON_VIOLATION, + DEFAULT_MAX_CPUS, DEFAULT_MAX_FILE_SIZE_KB, DEFAULT_TOOL_NAME, EXIT_CODE_FAIL_ON_VIOLATION, EXIT_CODE_INVALID_CONFIGURATION, EXIT_CODE_INVALID_DIRECTORY, EXIT_CODE_NO_DIRECTORY, EXIT_CODE_NO_OUTPUT, EXIT_CODE_RULESET_NOT_FOUND, EXIT_CODE_RULE_FILE_WITH_CONFIGURATION, - EXIT_CODE_UNSAFE_SUBDIRECTORIES, + EXIT_CODE_UNSAFE_SUBDIRECTORIES, SECRETS_HISTORY_TOOL_NAME, }; use cli::csv; use cli::datadog_utils::{ @@ -25,12 +25,14 @@ use cli::rule_utils::{ convert_secret_result_to_rule_result, count_violations_by_severities, get_languages_for_rules, get_rulesets_from_file, }; -use cli::sarif::sarif_utils::{generate_sarif_file, SarifReportMetadata}; +use cli::sarif::sarif_utils::{generate_sarif_file, HistoricalSecretResult, SarifReportMetadata}; use cli::utils::{choose_cpu_count, print_configuration}; use cli::violations_table; use common::analysis_options::AnalysisOptions; use common::model::diff_aware::DiffAware; -use datadog_static_analyzer::{secret_analysis, static_analysis, CliResults}; +use datadog_static_analyzer::{ + git_history_secret_analysis, secret_analysis, static_analysis, CliResults, +}; use kernel::analysis::ddsa_lib::v8_platform::{initialize_v8, Initialized, V8Platform}; use kernel::analysis::generated_content::DEFAULT_IGNORED_GLOBS; use kernel::classifiers::ArtifactClassification; @@ -40,7 +42,7 @@ use kernel::constants::{CARGO_VERSION, VERSION}; use kernel::model::common::OutputFormat; use kernel::model::rule::{Rule, RuleResult, RuleSeverity}; use kernel::rule_config::RuleConfigProvider; -use secrets::model::secret_result::SecretValidationStatus; +use secrets::model::secret_result::{SecretResult, SecretValidationStatus}; use secrets::secret_files::should_ignore_file_for_secret; use std::collections::HashMap; use std::io::prelude::*; @@ -154,6 +156,14 @@ fn main() -> Result<()> { "how long a rule can run before being killed, in milliseconds", "1000", ); + opts.optflag( + "", + "scan-git-history-only", + "scan the entire git history for secrets that are not present at the current HEAD. Every \ + commit is scanned (not just what current refs point to); a finding is 'history-only' \ + relative to HEAD, so a secret still live at the tip of another branch, tag, or remote may \ + also be reported as history-only.", + ); let matches = match opts.parse(&args[1..]) { Ok(m) => m, @@ -228,6 +238,15 @@ fn main() -> Result<()> { .unwrap_or(false); let secrets_enabled = secrets_enabled_old_option || secrets_enabled_new_option; + let scan_git_history_only = matches.opt_present("scan-git-history-only"); + + // A git-history scan produces a historic-only secrets report, so it is meaningless + // without secrets enabled. Fail fast rather than silently running a no-op. + if scan_git_history_only && !secrets_enabled { + eprintln!("--scan-git-history-only requires secrets scanning; pass --enable-secrets true"); + exit(EXIT_CODE_INVALID_CONFIGURATION); + } + let output_file = matches .opt_str("o") .context("output file must be specified")?; @@ -611,8 +630,9 @@ fn main() -> Result<()> { } // Secrets detection - - if secrets_enabled { + // + // Skipped entirely for a git-history scan. + if secrets_enabled && !scan_git_history_only { let secrets_start = Instant::now(); let secrets_files: Vec = files_in_repository @@ -671,6 +691,23 @@ fn main() -> Result<()> { result.secrets = Some(execution_results); } + // Git history scanning + // + // When `--scan-git-history-only` is set, the secrets results are replaced with the + // historical findings (the HEAD secret scan is skipped above). Static analysis is an + // independent product and is left untouched. + let mut historic_secrets: Vec = Vec::new(); + if scan_git_history_only { + let history_start = std::time::Instant::now(); + historic_secrets = git_history_secret_analysis(&configuration, &analysis_options) + .context("git history secret analysis failed")?; + + let history_duration = history_start.elapsed().as_secs_f64(); + println!("Git History Secrets Summary"); + println!(" Historical secrets found: {}", historic_secrets.len()); + println!(" Duration: {:.3}s", history_duration); + } + let global_execution_time_secs = global_start_time.elapsed().as_secs(); // if we have more than one static analysis violation and printing is enabled, show all @@ -686,6 +723,15 @@ fn main() -> Result<()> { .map(|r| r.rule_results) .unwrap_or_default(); + // Historic findings carry their commit provenance in `historic_secrets` (consumed by the SARIF + // output). For JSON/CSV, surface their base results alongside any HEAD secrets. HEAD and + // historic scans are mutually exclusive, so at most one of the two is non-empty. + let secrets_for_text_output: Vec = secrets_violations + .iter() + .cloned() + .chain(historic_secrets.iter().map(|h| h.inner.clone())) + .collect(); + let nb_total_static_analysis_violations: usize = static_analysis_rule_results .iter() .map(|x| x.violations.len()) @@ -705,7 +751,7 @@ fn main() -> Result<()> { let value = match configuration.output_format { OutputFormat::Csv => { - csv::generate_csv_results(&static_analysis_rule_results, &secrets_violations) + csv::generate_csv_results(&static_analysis_rule_results, &secrets_for_text_output) } OutputFormat::Json => { // make sure suppressed results are not included @@ -717,8 +763,7 @@ fn main() -> Result<()> { r }) .collect(); - let filtered_secrets: Vec = secrets_violations - .clone() + let filtered_secrets: Vec = secrets_for_text_output .iter() .map(convert_secret_result_to_rule_result) .map(|mut r| { @@ -733,12 +778,19 @@ fn main() -> Result<()> { &configuration, static_analysis_rule_results, secrets_violations, + historic_secrets, SarifReportMetadata { add_git_info, debug: configuration.use_debug, config_digest: configuration.generate_diff_aware_digest(), diff_aware_parameters, execution_time_secs: global_execution_time_secs, + tool_name: if scan_git_history_only { + SECRETS_HISTORY_TOOL_NAME.to_string() + } else { + DEFAULT_TOOL_NAME.to_string() + }, + split_runs_by_tool: scan_git_history_only, }, &all_path_metadata, ) diff --git a/crates/bins/src/git_history.rs b/crates/bins/src/git_history.rs new file mode 100644 index 00000000..70fa7777 --- /dev/null +++ b/crates/bins/src/git_history.rs @@ -0,0 +1,600 @@ +// Git history secret scanning: scan every unique blob in the object database +// (pass 1), then attribute only the secret-bearing blobs back to their +// introducing/removal commits via a single `git log` pass (pass 2). + +use cli::model::cli_configuration::CliConfiguration; +use cli::sarif::sarif_utils::HistoricalSecretResult; +use common::analysis_options::AnalysisOptions; +use git2::{ObjectType, Oid, Repository, TreeWalkMode, TreeWalkResult}; +use rayon::prelude::*; +use secrets::model::secret_result::SecretResult; +use secrets::scanner::{build_sds_scanner, find_secrets}; +use secrets::secret_files::should_ignore_file_for_secret; +use std::collections::{HashMap, HashSet}; +use std::path::Path; + +/// Placeholder path used when scanning a raw blob whose path is not yet known. +/// The secrets scanner only uses the filename for log/error messages, not for +/// matching, so any stable value works here. +const GIT_HISTORY_BLOB_PLACEHOLDER: &str = ""; + +/// An occurrence of a blob at a path in history, with its introducing commit +/// (where the blob was reachably added at that path) and, if applicable, the +/// commit that removed it from that path. Only occurrences with an +/// `introduced_at` are reported (a path known only via a deletion has its add on +/// an unreachable ancestor and must not produce a finding). +#[derive(Default)] +struct BlobOccurrence { + introduced_at: Option, + removed_at: Option, +} + +/// Get the `BlobOccurrence` for a (blob, path) pair, creating an empty one if it +/// does not exist yet. +fn find_or_create_occurrence<'a>( + registry: &'a mut HashMap>, + blob_oid: Oid, + path: &str, +) -> &'a mut BlobOccurrence { + registry + .entry(blob_oid) + .or_default() + .entry(path.to_string()) + .or_default() +} + +/// Collect all (blob_oid, path) pairs reachable from HEAD. +fn collect_head_blob_paths(repo: &Repository) -> HashSet<(Oid, String)> { + let mut set = HashSet::new(); + let head = repo.head().ok().and_then(|r| r.peel_to_commit().ok()); + if let Some(commit) = head { + if let Ok(tree) = commit.tree() { + tree.walk(TreeWalkMode::PreOrder, |dir, entry| { + if entry.kind() == Some(ObjectType::Blob) { + let path = if dir.is_empty() { + entry.name().unwrap_or("").to_string() + } else { + format!("{}{}", dir, entry.name().unwrap_or("")) + }; + set.insert((entry.id(), path)); + } + TreeWalkResult::Ok + }) + .ok(); + } + } + set +} + +/// Pass 1: scan every unique blob in the object database once for secrets. +/// +/// The object database is content-addressed, so this naturally deduplicates blob +/// content for free: there is no commit or tree walk. Every unique blob is scanned +/// (including blobs whose content also exists at HEAD): a blob present at HEAD may +/// also live at a historical path, and HEAD exclusion is applied per-(blob, path) in +/// pass 2. Returns the blobs that contained at least one secret, keyed by blob OID. +fn scan_all_blobs_for_secrets( + repo: &Repository, + config: &CliConfiguration, + options: &AnalysisOptions, +) -> anyhow::Result>> { + let secrets_rules = &config.secrets_rules; + let sds_scanner = + build_sds_scanner(secrets_rules, config.use_debug).map_err(|e| anyhow::anyhow!(e))?; + let max_blob_bytes = (config.max_file_size_kb * 1024) as usize; + + // Enumerate every object OID (the callback only yields the OID; cheap, reads + // the pack index without decompressing). + let t_enum = std::time::Instant::now(); + let mut all_oids: Vec = Vec::new(); + repo.odb()?.foreach(|oid| { + all_oids.push(*oid); + true + })?; + let total_objects = all_oids.len(); + eprintln!( + "[git-history] pass1: enumerated {} objects in {:.1}s; scanning blobs on {} threads", + total_objects, + t_enum.elapsed().as_secs_f64(), + config.get_num_threads(), + ); + + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + let objects_examined = AtomicU64::new(0); + let blobs_scanned = AtomicU64::new(0); + let secret_blobs = AtomicUsize::new(0); + let t_scan = std::time::Instant::now(); + let source_directory = config.source_directory.clone(); + + // git2 types are not `Send`, so each rayon worker opens (and reuses) its own + // `Repository` handle via a thread-local. + thread_local! { + static TL_REPO: std::cell::RefCell> = + const { std::cell::RefCell::new(None) }; + } + + let collected: Vec<(Oid, Vec)> = all_oids + .par_iter() + .filter_map(|&oid| { + let examined = objects_examined.fetch_add(1, Ordering::Relaxed) + 1; + if examined.is_multiple_of(1_000_000) { + let scanned = blobs_scanned.load(Ordering::Relaxed); + eprintln!( + "[git-history] pass1: examined {}/{} objects, scanned {} blobs ({:.0}/s), {} with secrets", + examined, + total_objects, + scanned, + scanned as f64 / t_scan.elapsed().as_secs_f64().max(0.001), + secret_blobs.load(Ordering::Relaxed), + ); + } + + // Read the blob content under a SHORT-LIVED borrow of the thread-local + // repo, then release the borrow before scanning. `find_secrets` re-enters + // the global rayon pool internally; if we held the `RefCell` borrow across + // it, a stolen sibling task on the same worker thread would re-enter this + // closure and panic with `already borrowed`. Copying the bytes out keeps + // the borrow scope free of any rayon re-entry. + let content: Option = TL_REPO.with(|cell| { + let mut slot = cell.borrow_mut(); + if slot.is_none() { + *slot = Repository::open(&source_directory).ok(); + } + let repo = slot.as_ref()?; + let odb = repo.odb().ok()?; + + // `read_header` reads only the object header (type + size) without + // decompressing, so non-blob and oversized objects are filtered cheaply. + let (size, kind) = odb.read_header(oid).ok()?; + if kind != ObjectType::Blob || size > max_blob_bytes { + return None; + } + let obj = odb.read(oid).ok()?; + // Copy the blob's text out so the borrow can be released before scanning. + // Decode lossily to mirror the HEAD scan (read_file): a blob with invalid + // UTF-8 bytes must still be scanned for secrets, not silently dropped. + Some(String::from_utf8_lossy(obj.data()).into_owned()) + }); + let content = content?; + + let scanned = blobs_scanned.fetch_add(1, Ordering::Relaxed) + 1; + if scanned.is_multiple_of(250_000) { + eprintln!( + "[git-history] pass1: scanned {} blobs ({:.0}/s), {} with secrets", + scanned, + scanned as f64 / t_scan.elapsed().as_secs_f64().max(0.001), + secret_blobs.load(Ordering::Relaxed), + ); + } + + let secrets = find_secrets( + &sds_scanner, + secrets_rules, + GIT_HISTORY_BLOB_PLACEHOLDER, + &content, + options, + ); + if secrets.is_empty() { + None + } else { + secret_blobs.fetch_add(1, Ordering::Relaxed); + Some((oid, secrets)) + } + }) + .collect(); + + let found: HashMap> = collected.into_iter().collect(); + eprintln!( + "[git-history] pass1 done: examined {} objects, scanned {} blobs in {:.1}s, {} secret-bearing blobs", + objects_examined.load(Ordering::Relaxed), + blobs_scanned.load(Ordering::Relaxed), + t_scan.elapsed().as_secs_f64(), + found.len(), + ); + + Ok(found) +} + +/// Pass 2: map a (small) set of secret-bearing blob OIDs back to the (path, +/// introducing commit) pairs where they appear across all refs (branches, tags, +/// remote-tracking branches, stash). +/// +/// git keeps no reverse index from blob to commits, so attribution requires a +/// history traversal. We shell out to a single `git log --all --raw` pass and +/// stream-parse it, keeping only deltas whose new blob OID is one of the (rare) +/// `targets`. `--all` walks every ref (branches, tags, remote-tracking branches, +/// stash), so a secret living only in a tagged release or a remote branch is still +/// attributed rather than silently dropped at emit time. git's diff machinery +/// (commit-graph + tree-OID subtree skipping) walks +/// the whole history in tens of seconds, where the equivalent libgit2 per-commit +/// `diff_tree_to_tree` loop is orders of magnitude slower. +/// +/// `git log` output is newest-first, so for each (blob, path) we overwrite the +/// recorded commit as we stream; the final value is the oldest (introducing) commit. +/// All distinct paths are collected per target blob (a blob may live at several paths). +fn attribute_blobs_to_paths( + source_directory: &str, + targets: &HashSet, +) -> anyhow::Result>> { + use std::io::BufRead; + + let mut registry: HashMap> = HashMap::new(); + if targets.is_empty() { + return Ok(registry); + } + // Hex strings for matching against the `git log --raw` blob OIDs. + let target_hex: HashSet = targets.iter().map(|o| o.to_string()).collect(); + + eprintln!( + "[git-history] pass2: attributing {} secret-bearing blobs via 'git log --raw'", + targets.len() + ); + let t_walk = std::time::Instant::now(); + + let args = [ + "-c", + "core.quotePath=false", + "log", + "--all", + "--raw", + "--no-renames", + "--no-abbrev", + "--format=C %H", + ]; + + let mut child = std::process::Command::new("git") + .current_dir(source_directory) + .args(args) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn()?; + + let stdout = child.stdout.take().expect("git stdout is piped"); + let reader = std::io::BufReader::new(stdout); + + let mut current_commit: Option = None; + let mut commit_count: u64 = 0; + for line in reader.lines() { + let line = line?; + // Commit marker line: "C ". + if let Some(sha) = line.strip_prefix("C ") { + current_commit = Oid::from_str(sha.trim()).ok(); + commit_count += 1; + if commit_count.is_multiple_of(200_000) { + eprintln!( + "[git-history] pass2: parsed {} commits ({:.0}/s), {}/{} target blobs attributed", + commit_count, + commit_count as f64 / t_walk.elapsed().as_secs_f64().max(0.001), + registry.len(), + targets.len(), + ); + } + continue; + } + // Raw diff line: ": \t". + let Some((meta, path)) = line.split_once('\t') else { + continue; + }; + if !meta.starts_with(':') { + continue; + } + let fields: Vec<&str> = meta.split_whitespace().collect(); + if fields.len() < 5 { + continue; + } + let Some(commit) = current_commit else { + continue; + }; + let (old_sha, new_sha) = (fields[2], fields[3]); + + // NEW side (add/modify): the blob is present at this path as of this commit. + // The stream is newest-first, so overwriting keeps the OLDEST such commit as + // the introducing commit. A finding is only ever reported for a path with an + // introducing commit (a reachable add). + if target_hex.contains(new_sha) { + if let Ok(blob_oid) = Oid::from_str(new_sha) { + let occ = find_or_create_occurrence(&mut registry, blob_oid, path); + occ.introduced_at = Some(commit); + } + } + // OLD side (delete/modify): the blob left this path at this commit. Keep the + // FIRST seen (newest) as the removal commit. This only ever ENRICHES an + // occurrence with a removal commit; occurrences that never get an + // introducing commit are dropped at emit time, so this does not resurrect + // unreachable-origin paths. + if target_hex.contains(old_sha) { + if let Ok(blob_oid) = Oid::from_str(old_sha) { + let occ = find_or_create_occurrence(&mut registry, blob_oid, path); + if occ.removed_at.is_none() { + occ.removed_at = Some(commit); + } + } + } + } + + let status = child.wait()?; + if !status.success() { + return Err(anyhow::anyhow!( + "`git log` exited with status {status} during history attribution" + )); + } + + eprintln!( + "[git-history] pass2 done: parsed {} commits in {:.1}s, attributed {}/{} blobs", + commit_count, + t_walk.elapsed().as_secs_f64(), + registry.len(), + targets.len(), + ); + + Ok(registry) +} + +/// Scan git history for secrets that are not present at the current branch HEAD. +/// +/// Two passes: +/// 1. scan every unique blob in the object database for secrets (scan_all_blobs_for_secrets) +/// 2. for the rare blobs that contained a secret, reverse-map them to their (path, introducing commit) occurrences (attribute_blobs_to_paths). +/// +/// HEAD findings are already covered by the normal secret_analysis(), so (blob, path) pairs present at HEAD are excluded here. +pub fn git_history_secret_analysis( + config: &CliConfiguration, + options: &AnalysisOptions, +) -> anyhow::Result> { + let repo = Repository::open(&config.source_directory)?; + + // Pass 1: scan all unique blobs. + let secret_blobs = scan_all_blobs_for_secrets(&repo, config, options)?; + if secret_blobs.is_empty() { + return Ok(Vec::new()); + } + + // Pass 2: attribute the secret-bearing blobs to their (path, commit) pairs. + let targets: HashSet = secret_blobs.keys().copied().collect(); + let registry = attribute_blobs_to_paths(&config.source_directory, &targets)?; + let head_blob_paths = collect_head_blob_paths(&repo); + + let mut results: Vec = Vec::new(); + for (blob_oid, occurrences) in ®istry { + let Some(secrets) = secret_blobs.get(blob_oid) else { + continue; + }; + for (path, occurrence) in occurrences { + // Only report paths with a reachable introducing commit. An occurrence + // seen only via a deletion (no reachable add) is skipped. + let Some(introduced_at) = occurrence.introduced_at else { + continue; + }; + // Skip pairs present at HEAD at the same path (covered by normal scan). + if head_blob_paths.contains(&(*blob_oid, path.clone())) { + continue; + } + // Per-path file filtering (deferred from pass 1, which had no path). + if should_ignore_file_for_secret(Path::new(path)) { + continue; + } + for secret in secrets { + results.push(HistoricalSecretResult { + inner: secret.clone_with_path(path), + introducing_commit_sha: introduced_at, + removed_at_sha: occurrence.removed_at, + }); + } + } + } + + Ok(results) +} + +#[cfg(test)] +mod git_history_tests { + use super::*; + use git2::{Oid, Repository}; + use std::collections::HashSet; + use std::path::Path; + + fn init_repo(path: &Path) -> Repository { + Repository::init(path).unwrap() + } + + /// Write `files` (path, content), stage `deletes`, and create a commit on HEAD. + fn commit(repo: &Repository, files: &[(&str, &str)], deletes: &[&str], msg: &str) -> Oid { + let workdir = repo.workdir().unwrap().to_path_buf(); + let mut index = repo.index().unwrap(); + for (p, content) in files { + let full = workdir.join(p); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&full, content).unwrap(); + index.add_path(Path::new(p)).unwrap(); + } + for p in deletes { + let _ = std::fs::remove_file(workdir.join(p)); + index.remove_path(Path::new(p)).unwrap(); + } + index.write().unwrap(); + let tree = repo.find_tree(index.write_tree().unwrap()).unwrap(); + let sig = git2::Signature::now("Test", "test@example.com").unwrap(); + let parent = repo.head().ok().and_then(|h| h.peel_to_commit().ok()); + let parents: Vec<&git2::Commit> = parent.iter().collect(); + repo.commit(Some("HEAD"), &sig, &sig, msg, &tree, &parents) + .unwrap() + } + + fn blob_oid(repo: &Repository, content: &str) -> Oid { + repo.blob(content.as_bytes()).unwrap() + } + + fn config_with_rule(repo_dir: &str) -> CliConfiguration { + use kernel::config::common::PathConfig; + use kernel::model::common::OutputFormat; + use kernel::rule_config::RuleConfigProvider; + use secrets::model::secret_rule::{RulePriority, SecretRule}; + + let rule = SecretRule { + id: "test-rule".to_string(), + name: "test-rule".to_string(), + sds_id: "71A7A0ED-DD03-45C5-9C2E-56B30CB566E0".to_string(), + description: "test".to_string(), + pattern: "SECRETVALUE[0-9]+".to_string(), + default_included_keywords: vec![], + default_excluded_keywords: vec![], + look_ahead_character_count: Some(30), + priority: RulePriority::Medium, + validators: Some(vec![]), + validators_v2: None, + match_validation: None, + pattern_capture_groups: vec![], + is_supporting_rule: false, + }; + CliConfiguration { + use_debug: false, + configuration_method: None, + ignore_gitignore: true, + source_directory: repo_dir.to_string(), + source_subdirectories: vec![], + path_config: PathConfig::default(), + rules_file: None, + output_format: OutputFormat::Sarif, + output_file: String::new(), + num_cpus: 2, + rules: vec![], + rule_config_provider: RuleConfigProvider::default(), + max_file_size_kb: 200, + use_staging: false, + show_performance_statistics: false, + ignore_generated_files: false, + static_analysis_enabled: false, + secrets_enabled: true, + secrets_rules: vec![rule], + should_verify_checksum: false, + debug_java_dfa: false, + } + } + + /// A blob removed before HEAD is attributed to the single path/commit that + /// introduced it. + #[test] + fn attribute_single_path() { + let dir = tempfile::tempdir().unwrap(); + let repo = init_repo(dir.path()); + let c1 = commit(&repo, &[("config.txt", "SECRETVALUE123\n")], &[], "add"); + let secret = blob_oid(&repo, "SECRETVALUE123\n"); + commit(&repo, &[("config.txt", "clean\n")], &[], "scrub"); + + let targets: HashSet = [secret].into_iter().collect(); + let reg = attribute_blobs_to_paths(dir.path().to_str().unwrap(), &targets).unwrap(); + let occ = reg.get(&secret).expect("secret blob attributed"); + assert_eq!(occ.len(), 1); + let (path, occurrence) = occ.iter().next().unwrap(); + assert_eq!(path, "config.txt"); + assert_eq!(occurrence.introduced_at, Some(c1)); + } + + /// The same content at two different paths is reported at BOTH paths, each with + /// its own introducing commit. + #[test] + fn attribute_multi_path() { + let dir = tempfile::tempdir().unwrap(); + let repo = init_repo(dir.path()); + let c1 = commit(&repo, &[("a.txt", "DUP123\n")], &[], "add a"); + let c2 = commit(&repo, &[("b.txt", "DUP123\n")], &[], "add b"); + let dup = blob_oid(&repo, "DUP123\n"); + + let targets: HashSet = [dup].into_iter().collect(); + let reg = attribute_blobs_to_paths(dir.path().to_str().unwrap(), &targets).unwrap(); + let occ = reg.get(&dup).unwrap(); + let mut paths: Vec<_> = occ + .iter() + .map(|(p, o)| (p.as_str(), o.introduced_at)) + .collect(); + paths.sort(); + assert_eq!(paths, vec![("a.txt", Some(c1)), ("b.txt", Some(c2))]); + } + + /// One blob (empty content) duplicated across many paths in a single (root) + /// commit: all paths are recorded, no quadratic blowup, no panic. + #[test] + fn attribute_high_fanout() { + let dir = tempfile::tempdir().unwrap(); + let repo = init_repo(dir.path()); + let files: Vec<(String, String)> = (0..50) + .map(|i| (format!("d{i}/__init__.py"), String::new())) + .collect(); + let refs: Vec<(&str, &str)> = files + .iter() + .map(|(p, c)| (p.as_str(), c.as_str())) + .collect(); + let c1 = commit(&repo, &refs, &[], "many empty files"); + let empty = blob_oid(&repo, ""); + + let targets: HashSet = [empty].into_iter().collect(); + let reg = attribute_blobs_to_paths(dir.path().to_str().unwrap(), &targets).unwrap(); + let occ = reg.get(&empty).unwrap(); + assert_eq!(occ.len(), 50); + assert!(occ.values().all(|o| o.introduced_at == Some(c1))); + } + + /// A moved file (delete old path + add new path) is attributed at BOTH paths: + /// the new path via the add, the old path via the delete (matching the delta's + /// OLD blob OID). The old path keeps its true introducing commit. + #[test] + fn attribute_move_records_both_paths() { + let dir = tempfile::tempdir().unwrap(); + let repo = init_repo(dir.path()); + let c1 = commit(&repo, &[("a.txt", "MOVED123\n")], &[], "add a"); + let c2 = commit(&repo, &[("b.txt", "MOVED123\n")], &["a.txt"], "move a -> b"); + let blob = blob_oid(&repo, "MOVED123\n"); + + let targets: HashSet = [blob].into_iter().collect(); + let reg = attribute_blobs_to_paths(dir.path().to_str().unwrap(), &targets).unwrap(); + let occ = reg.get(&blob).expect("moved blob attributed"); + let mut paths: Vec<_> = occ + .iter() + .map(|(p, o)| (p.as_str(), o.introduced_at, o.removed_at)) + .collect(); + paths.sort(); + // a.txt: introduced at c1 (oldest add wins over the c2 delete sighting) and + // removed at c2 (the move deletes it). b.txt: introduced at c2, still present. + assert_eq!( + paths, + vec![("a.txt", Some(c1), Some(c2)), ("b.txt", Some(c2), None),] + ); + } + + /// End-to-end: a secret removed before HEAD is reported as historical with the + /// correct introducing SHA, while a secret still present at HEAD (same blob+path) + /// is NOT reported (it is covered by the normal HEAD scan). + #[test] + fn history_scan_excludes_head_and_tags_historical() { + let dir = tempfile::tempdir().unwrap(); + let repo = init_repo(dir.path()); + let c1 = commit( + &repo, + &[ + ("secret.txt", "SECRETVALUE123\n"), + ("keep.txt", "SECRETVALUE999\n"), + ], + &[], + "add secrets", + ); + // Scrub secret.txt; keep.txt (with its secret) stays at HEAD. + let c2 = commit(&repo, &[("secret.txt", "clean\n")], &[], "scrub"); + + let config = config_with_rule(dir.path().to_str().unwrap()); + let options = AnalysisOptions::default(); + let results = git_history_secret_analysis(&config, &options).unwrap(); + + assert_eq!(results.len(), 1, "only the removed secret is historical"); + let r = &results[0]; + assert_eq!(r.inner.filename, "secret.txt"); + assert_eq!(r.introducing_commit_sha, c1); + // secret.txt's secret was scrubbed in c2, so it is the removal commit. + assert_eq!(r.removed_at_sha, Some(c2)); + assert!( + results.iter().all(|x| x.inner.filename != "keep.txt"), + "a secret still present at HEAD must not be reported as historical" + ); + } +} diff --git a/crates/bins/src/lib.rs b/crates/bins/src/lib.rs index 95a6c78c..21adcc52 100644 --- a/crates/bins/src/lib.rs +++ b/crates/bins/src/lib.rs @@ -22,6 +22,9 @@ use std::process::exit; use std::sync::Arc; use std::time::Duration; +mod git_history; +pub use git_history::git_history_secret_analysis; + /// Read a file and if the file has some invalid UTF-8 characters, it returns a string with invalid /// characters. pub fn read_file(path: &Path) -> anyhow::Result { diff --git a/crates/cli/src/constants.rs b/crates/cli/src/constants.rs index 30995cc3..84a5d1da 100644 --- a/crates/cli/src/constants.rs +++ b/crates/cli/src/constants.rs @@ -8,9 +8,13 @@ pub static HEADER_CONTENT_TYPE: &str = "Content-Type"; pub static HEADER_CONTENT_TYPE_APPLICATION_JSON: &str = "application/json"; pub static HEADER_USER_AGENT: &str = "User-Agent"; pub static USER_AGENT_PRODUCT: &str = "datadog-static-analyzer"; +pub static DEFAULT_TOOL_NAME: &str = "datadog-static-analyzer"; +pub static SECRETS_HISTORY_TOOL_NAME: &str = "datadog-static-analyzer-secrets-history"; pub static QUERY_PARAM_SCHEMA_VERSION: &str = "schema_version"; pub static SARIF_PROPERTY_DATADOG_FINGERPRINT: &str = "DATADOG_FINGERPRINT"; pub static SARIF_PROPERTY_SHA: &str = "SHA"; +pub static SARIF_PROPERTY_IS_GIT_HISTORY_ONLY: &str = "isGitHistoryOnly"; +pub static SARIF_PROPERTY_REMOVED_AT_SHA: &str = "removedAtSha"; pub static DEFAULT_MAX_CPUS: usize = 8; pub static DEFAULT_MAX_FILE_SIZE_KB: u64 = 200; // See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html diff --git a/crates/cli/src/sarif/sarif_utils.rs b/crates/cli/src/sarif/sarif_utils.rs index 22cf71f1..0553781e 100644 --- a/crates/cli/src/sarif/sarif_utils.rs +++ b/crates/cli/src/sarif/sarif_utils.rs @@ -2,12 +2,15 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::Path; use std::rc::Rc; -use crate::constants::{SARIF_PROPERTY_DATADOG_FINGERPRINT, SARIF_PROPERTY_SHA}; +use crate::constants::{ + DEFAULT_TOOL_NAME, SARIF_PROPERTY_DATADOG_FINGERPRINT, SARIF_PROPERTY_IS_GIT_HISTORY_ONLY, + SARIF_PROPERTY_REMOVED_AT_SHA, SARIF_PROPERTY_SHA, SECRETS_HISTORY_TOOL_NAME, +}; use anyhow::Result; use base64::Engine; use common::model::position::Position; use common::model::position::PositionBuilder; -use git2::{BlameOptions, Repository}; +use git2::{BlameOptions, Oid, Repository}; use kernel::classifiers::ArtifactClassification; use kernel::constants::CARGO_VERSION; use kernel::model::rule::{RuleCategory, RuleSeverity}; @@ -42,12 +45,18 @@ trait IntoSarif { /// The `SarifReportMetadata` structure contains all metadata being added to the sarif report. /// Those metadata is being added as property is being used to enhance the generation /// of the SARIF report. +#[derive(Clone)] pub struct SarifReportMetadata { pub add_git_info: bool, pub debug: bool, pub config_digest: String, pub diff_aware_parameters: Option, pub execution_time_secs: u64, + pub tool_name: String, + /// When true, static-analysis and secrets findings are emitted as separate SARIF runs (each + /// under its own tool driver name). When false, they share a single concatenated run. This is + /// independent of `tool_name`, which only sets the driver label. + pub split_runs_by_tool: bool, } #[derive(Debug, Clone)] @@ -121,6 +130,18 @@ impl From for SarifRule { } } +/// A secret found only in git history. Wraps the base [`SecretResult`] with the +/// commits that introduced and (optionally) removed it. Owned by the SARIF layer +/// so the `secrets` model stays free of git-history concepts. `introducing_commit_sha` +/// is non-optional: a `HistoricalSecretResult` is only built once the introducing +/// commit is known. +#[derive(Debug, Clone)] +pub struct HistoricalSecretResult { + pub inner: SecretResult, + pub introducing_commit_sha: Oid, + pub removed_at_sha: Option, +} + /// Generic representation of a violation for both static analysis and secrets #[derive(Debug, Clone)] pub enum SarifViolation { @@ -164,6 +185,38 @@ impl SarifViolation { pub enum SarifRuleResult { StaticAnalysis(RuleResult), Secret(SecretResult), + HistoricalSecret(HistoricalSecretResult), +} + +/// Build the per-match [`SarifViolation`]s for a secret result (HEAD or historical). +fn secret_violations(secret_result: &SecretResult) -> Vec { + secret_result + .matches + .iter() + .map(|r| { + let severity = match &r.validation_status { + SecretValidationStatus::NotValidated => RuleSeverity::Notice, + SecretValidationStatus::Valid => RuleSeverity::Error, + SecretValidationStatus::Invalid => RuleSeverity::None, + SecretValidationStatus::ValidationError(_) => RuleSeverity::Warning, + SecretValidationStatus::NotAvailable => RuleSeverity::Error, + }; + + Secret( + Violation { + start: r.start, + end: r.end, + message: secret_result.message.clone(), + severity, + category: RuleCategory::Security, + fixes: vec![], + taint_flow: None, + is_suppressed: r.is_suppressed, + }, + r.validation_status.clone(), + ) + }) + .collect::>() } impl SarifRuleResult { @@ -174,33 +227,8 @@ impl SarifRuleResult { .iter() .map(|v| StaticAnalysis(v.clone())) .collect::>(), - SarifRuleResult::Secret(secret_result) => secret_result - .matches - .iter() - .map(|r| { - let severity = match &r.validation_status { - SecretValidationStatus::NotValidated => RuleSeverity::Notice, - SecretValidationStatus::Valid => RuleSeverity::Error, - SecretValidationStatus::Invalid => RuleSeverity::None, - SecretValidationStatus::ValidationError(_) => RuleSeverity::Warning, - SecretValidationStatus::NotAvailable => RuleSeverity::Error, - }; - - Secret( - Violation { - start: r.start, - end: r.end, - message: secret_result.message.clone(), - severity, - category: RuleCategory::Security, - fixes: vec![], - taint_flow: None, - is_suppressed: r.is_suppressed, - }, - r.validation_status.clone(), - ) - }) - .collect::>(), + SarifRuleResult::Secret(secret_result) => secret_violations(secret_result), + SarifRuleResult::HistoricalSecret(h) => secret_violations(&h.inner), } } @@ -209,6 +237,7 @@ impl SarifRuleResult { match self { SarifRuleResult::StaticAnalysis(r) => &r.filename, SarifRuleResult::Secret(r) => &r.filename, + SarifRuleResult::HistoricalSecret(h) => &h.inner.filename, } } @@ -217,6 +246,7 @@ impl SarifRuleResult { as_slash_path(match self { SarifRuleResult::StaticAnalysis(r) => &r.filename, SarifRuleResult::Secret(r) => &r.filename, + SarifRuleResult::HistoricalSecret(h) => &h.inner.filename, }) } @@ -224,6 +254,7 @@ impl SarifRuleResult { match self { SarifRuleResult::StaticAnalysis(r) => r.rule_name.as_str(), SarifRuleResult::Secret(r) => r.rule_name.as_str(), + SarifRuleResult::HistoricalSecret(h) => h.inner.rule_name.as_str(), } } @@ -231,6 +262,7 @@ impl SarifRuleResult { match self { SarifRuleResult::StaticAnalysis(r) => r.rule_name.as_str(), SarifRuleResult::Secret(r) => r.rule_id.as_str(), + SarifRuleResult::HistoricalSecret(h) => h.inner.rule_id.as_str(), } } } @@ -271,6 +303,7 @@ pub struct SarifGenerationOptions { pub diff_aware_parameters: Option, pub repository_directory: String, pub execution_time_secs: u64, + pub tool_name: String, } impl IntoSarif for &SecretRule { @@ -465,7 +498,7 @@ fn generate_tool_section(rules: &[SarifRule], options: &SarifGenerationOptions) } let driver: ToolComponent = ToolComponentBuilder::default() - .name("datadog-static-analyzer") + .name(options.tool_name.as_str()) .version(CARGO_VERSION) .information_uri("https://www.datadoghq.com") .rules( @@ -727,14 +760,21 @@ fn generate_results( .transpose(); let taint_code_flow = taint_code_flow?; - let sha_option = if options.add_git_info { - get_sha_for_line( + // For historical findings, use the introducing commit SHA + // instead of git blame (file may not exist at HEAD). + let historical = match rule_result { + SarifRuleResult::HistoricalSecret(h) => Some(h), + _ => None, + }; + + let sha_option = match historical { + Some(h) => Some(h.introducing_commit_sha.to_string()), + None if options.add_git_info => get_sha_for_line( &rule_result.slash_path_str(), violation.start.line as usize, &options, - ) - } else { - None + ), + None => None, }; let fingerprint_option = get_fingerprint_for_violation( @@ -745,7 +785,7 @@ fn generate_results( options.debug, ); - let partial_fingerprints: BTreeMap = + let mut partial_fingerprints: BTreeMap = match (sha_option, fingerprint_option) { (Some(sha), Some(fp)) => BTreeMap::from([ (SARIF_PROPERTY_SHA.to_string(), sha), @@ -761,6 +801,19 @@ fn generate_results( _ => BTreeMap::new(), }; + if let Some(h) = historical { + partial_fingerprints.insert( + SARIF_PROPERTY_IS_GIT_HISTORY_ONLY.to_string(), + "true".to_string(), + ); + if let Some(removed_at) = h.removed_at_sha { + partial_fingerprints.insert( + SARIF_PROPERTY_REMOVED_AT_SHA.to_string(), + removed_at.to_string(), + ); + } + } + let mut sarif_result = result_builder.clone(); sarif_result @@ -888,6 +941,7 @@ pub fn generate_sarif_report( diff_aware_parameters: tool_information.diff_aware_parameters.clone(), repository_directory: directory.clone(), execution_time_secs: tool_information.execution_time_secs, + tool_name: tool_information.tool_name.clone(), }; let artifacts_kv = extract_artifacts( @@ -914,6 +968,7 @@ pub fn generate_sarif_file( configuration: &CliConfiguration, static_analysis_rule_results: Vec, secrets_rule_results: Vec, + historic_secret_results: Vec, sarif_report_metadata: SarifReportMetadata, path_metadata: &HashMap, ) -> Result { @@ -934,24 +989,77 @@ pub fn generate_sarif_file( .map(SarifRuleResult::try_from) .collect::, _>>() .map_err(anyhow::Error::msg)?; - let secret_results = secrets_rule_results + let mut secret_results = secrets_rule_results .into_iter() .map(SarifRuleResult::try_from) .collect::, _>>() .map_err(anyhow::Error::msg)?; + // Historic secrets share the secrets run/rules; they carry their own commit provenance. + secret_results.extend( + historic_secret_results + .into_iter() + .map(SarifRuleResult::HistoricalSecret), + ); - match generate_sarif_report( - &[static_rules_sarif, secrets_rules_sarif].concat(), - &[static_analysis_results, secret_results].concat(), - &configuration.source_directory, - sarif_report_metadata, - path_metadata, - ) { - Ok(report) => { - Ok(serde_json::to_string(&report).expect("error when getting the SARIF report")) + // In normal mode both kinds of findings share a single concatenated run. In git-history mode + // the caller requests split runs so static-analysis and historic-secret findings are each + // attributed to their own tool driver. + let report = if !sarif_report_metadata.split_runs_by_tool { + generate_sarif_report( + &[static_rules_sarif, secrets_rules_sarif].concat(), + &[static_analysis_results, secret_results].concat(), + &configuration.source_directory, + sarif_report_metadata, + path_metadata, + )? + } else { + merge_sarif_runs( + &static_rules_sarif, + &static_analysis_results, + &secrets_rules_sarif, + &secret_results, + &configuration.source_directory, + &sarif_report_metadata, + path_metadata, + )? + }; + + Ok(serde_json::to_string(&report).expect("error when getting the SARIF report")) +} + +/// Builds a SARIF report with up to two runs: a static-analysis run named [`DEFAULT_TOOL_NAME`] +/// and a historic-secrets run named [`SECRETS_HISTORY_TOOL_NAME`]. A group with no results is +/// skipped so we never emit an empty run. +fn merge_sarif_runs( + static_rules: &[SarifRule], + static_results: &[SarifRuleResult], + secrets_rules: &[SarifRule], + secret_results: &[SarifRuleResult], + source_directory: &String, + base_metadata: &SarifReportMetadata, + path_metadata: &HashMap, +) -> Result { + let groups = [ + (static_rules, static_results, DEFAULT_TOOL_NAME), + (secrets_rules, secret_results, SECRETS_HISTORY_TOOL_NAME), + ]; + + let mut runs = vec![]; + for (rules, results, tool_name) in groups { + if results.is_empty() { + continue; } - Err(err) => Err(err), + let mut metadata = base_metadata.clone(); + metadata.tool_name = tool_name.to_string(); + let report = + generate_sarif_report(rules, results, source_directory, metadata, path_metadata)?; + runs.extend(report.runs); } + + Ok(SarifBuilder::default() + .version("2.1.0") + .runs(runs) + .build()?) } /// Returns the file path for this result as a slash path, a path whose components are. @@ -1176,6 +1284,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1298,6 +1408,191 @@ mod tests { assert!(validate_data(&sarif_report_to_string)); } + // The `tool_name` from the report metadata is surfaced as the SARIF tool driver name. + // This is what lets a git-history scan produce a distinctly-named report. + #[test] + fn test_tool_driver_name_from_metadata() { + for name in [ + crate::constants::DEFAULT_TOOL_NAME, + crate::constants::SECRETS_HISTORY_TOOL_NAME, + ] { + let sarif_report = generate_sarif_report( + &[], + &vec![], + &"mydir".to_string(), + SarifReportMetadata { + add_git_info: false, + debug: false, + config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), + diff_aware_parameters: None, + execution_time_secs: 42, + tool_name: name.to_string(), + split_runs_by_tool: false, + }, + &Default::default(), + ) + .expect("generate sarif report"); + + let sarif_json = serde_json::to_value(sarif_report).unwrap(); + assert_eq!( + sarif_json + .pointer("/runs/0/tool/driver/name") + .and_then(|v| v.as_str()), + Some(name) + ); + } + } + + /// A git-history report keeps static-analysis and historic-secret findings in two separate + /// runs, each named after its own tool. A group with no results is not emitted. Normal mode + /// keeps everything in a single `datadog-static-analyzer` run. + #[test] + fn test_git_history_report_splits_runs_by_tool() { + let static_rule: SarifRule = RuleBuilder::default() + .name("my-rule".to_string()) + .description_base64(None) + .language(Language::Python) + .checksum("blabla".to_string()) + .pattern(None) + .tree_sitter_query_base64(None) + .category(RuleCategory::BestPractices) + .code_base64("Zm9vYmFyYmF6".to_string()) + .short_description_base64(None) + .entity_checked(None) + .rule_type(RuleType::TreeSitterQuery) + .severity(RuleSeverity::Error) + .cwe(None) + .arguments(vec![]) + .tests(vec![]) + .is_testing(false) + .documentation_url(None) + .build() + .unwrap() + .into(); + let static_result: SarifRuleResult = RuleResultBuilder::default() + .rule_name("my-rule".to_string()) + .filename("myfile.py".to_string()) + .violations(vec![ViolationBuilder::default() + .start(PositionBuilder::default().line(1).col(2).build().unwrap()) + .end(PositionBuilder::default().line(3).col(4).build().unwrap()) + .message("violation message".to_string()) + .severity(RuleSeverity::Error) + .category(RuleCategory::BestPractices) + .fixes(vec![]) + .taint_flow(None) + .build() + .unwrap()]) + .output(None) + .errors(vec![]) + .execution_time_ms(0) + .parsing_time_ms(0) + .query_node_time_ms(0) + .execution_error(None) + .build() + .unwrap() + .try_into() + .unwrap(); + + let secret_rule: SarifRule = secrets::model::secret_rule::SecretRule { + id: "secret-rule".to_string(), + name: "secret-rule".to_string(), + sds_id: "71A7A0ED-DD03-45C5-9C2E-56B30CB566E0".to_string(), + description: "secret-description".to_string(), + pattern: "foobarbaz".to_string(), + priority: RulePriority::Medium, + default_included_keywords: vec![], + default_excluded_keywords: vec![], + look_ahead_character_count: Some(30), + validators: Some(vec![]), + validators_v2: None, + match_validation: None, + pattern_capture_groups: vec![], + is_supporting_rule: false, + } + .into(); + let secret_result: SarifRuleResult = + SarifRuleResult::HistoricalSecret(HistoricalSecretResult { + inner: SecretResult { + rule_id: "secret-rule".to_string(), + rule_name: "secret-rule".to_string(), + filename: "myfile.py".to_string(), + message: "some secret".to_string(), + priority: RulePriority::Medium, + matches: vec![SecretResultMatch { + start: Position { line: 1, col: 1 }, + end: Position { line: 2, col: 2 }, + validation_status: SecretValidationStatus::NotValidated, + is_suppressed: false, + }], + }, + introducing_commit_sha: Oid::from_str("abc1230000000000000000000000000000000000") + .unwrap(), + removed_at_sha: None, + }); + + let base_metadata = || SarifReportMetadata { + add_git_info: false, + debug: false, + config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), + diff_aware_parameters: None, + execution_time_secs: 42, + tool_name: crate::constants::SECRETS_HISTORY_TOOL_NAME.to_string(), + split_runs_by_tool: true, + }; + + let run_tool_names = |report: &Sarif| -> Vec { + report + .runs + .iter() + .map(|r| r.tool.driver.name.clone()) + .collect() + }; + + // Both kinds present -> two runs, each named after its own tool. + let report = merge_sarif_runs( + std::slice::from_ref(&static_rule), + std::slice::from_ref(&static_result), + std::slice::from_ref(&secret_rule), + std::slice::from_ref(&secret_result), + &"mydir".to_string(), + &base_metadata(), + &Default::default(), + ) + .expect("merge sarif runs"); + let names = run_tool_names(&report); + assert_eq!(names.len(), 2); + assert!(names.iter().any(|n| n == DEFAULT_TOOL_NAME)); + assert!(names.iter().any(|n| n == SECRETS_HISTORY_TOOL_NAME)); + assert!(validate_data(&serde_json::to_value(&report).unwrap())); + + // Only historic secrets present -> a single secrets-history run (empty static group is skipped). + let report = merge_sarif_runs( + &[], + &[], + std::slice::from_ref(&secret_rule), + std::slice::from_ref(&secret_result), + &"mydir".to_string(), + &base_metadata(), + &Default::default(), + ) + .expect("merge sarif runs"); + assert_eq!(run_tool_names(&report), vec![SECRETS_HISTORY_TOOL_NAME]); + + // Normal mode keeps everything under a single default-named run. + let report = generate_sarif_report( + &[static_rule, secret_rule], + &[static_result, secret_result], + &"mydir".to_string(), + SarifReportMetadata { + tool_name: DEFAULT_TOOL_NAME.to_string(), + ..base_metadata() + }, + &Default::default(), + ) + .expect("generate sarif report"); + assert_eq!(run_tool_names(&report), vec![DEFAULT_TOOL_NAME]); + } + // Ensure that diff-aware scanning information are correctly surfaced #[test] fn test_generate_sarif_diff_aware_scanning() { @@ -1316,6 +1611,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: Some(diff_aware_infos), execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1413,6 +1710,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1519,6 +1818,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1655,6 +1956,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1744,6 +2047,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1837,6 +2142,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1923,6 +2230,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -1989,6 +2298,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) @@ -2087,6 +2398,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 42, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &path_metadata, ) @@ -2187,6 +2500,8 @@ mod tests { config_digest: "5d7273dec32b80788b4d3eac46c866f0".to_string(), diff_aware_parameters: None, execution_time_secs: 0, + tool_name: crate::constants::DEFAULT_TOOL_NAME.to_string(), + split_runs_by_tool: false, }, &Default::default(), ) diff --git a/crates/secrets/src/model/secret_result.rs b/crates/secrets/src/model/secret_result.rs index 236cd985..b60d6d80 100644 --- a/crates/secrets/src/model/secret_result.rs +++ b/crates/secrets/src/model/secret_result.rs @@ -76,6 +76,15 @@ pub struct SecretResult { pub matches: Vec, } +impl SecretResult { + /// Clone with a different file path (used for blob fan-out in history scanning). + pub fn clone_with_path(&self, new_path: &str) -> Self { + let mut cloned = self.clone(); + cloned.filename = new_path.to_string(); + cloned + } +} + #[derive(Debug, PartialEq, PartialOrd, Ord, Eq, Clone, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ValidationErrorInfo {