diff --git a/provn-cli/src/diff.rs b/provn-cli/src/diff.rs index e16f4eb..6a36a71 100644 --- a/provn-cli/src/diff.rs +++ b/provn-cli/src/diff.rs @@ -32,30 +32,39 @@ pub fn parse_staged_diff(cfg: &Config) -> Result, DiffError> { } pub fn parse_file(path: &str, cfg: &Config) -> Result, DiffError> { + let file_path = PathBuf::from(path); + + if should_skip(&file_path, cfg) { + return Ok(vec![]); + } + let content = std::fs::read_to_string(path)?; - let ext = PathBuf::from(path) + + // Honour provn:skip-file anywhere in the file — same semantics as in diff mode. + if content.contains("provn:skip-file") || content.contains("aegis:skip-file") { + return Ok(vec![]); + } + + let ext = file_path .extension() .map(|e| e.to_string_lossy().to_string()) .unwrap_or_default(); - // Synthesize fake added lines (all lines are "added" when checking a file directly) + // Synthesize "added" lines from every line in the file, applying the same + // provn:allow filter that parse_diff_text uses so the annotation works in + // both `provn scan` (pre-commit) and `provn check` (file) modes. let added_lines: Vec<(usize, String)> = content .lines() .enumerate() + .filter(|(_, l)| !l.contains("provn:allow") && !l.contains("aegis:allow")) .map(|(i, l)| (i + 1, l.to_string())) .collect(); - let chunk = DiffChunk { - file: PathBuf::from(path), + Ok(vec![DiffChunk { + file: file_path, extension: ext, added_lines, - }; - - if should_skip(&chunk.file, cfg) { - return Ok(vec![]); - } - - Ok(vec![chunk]) + }]) } fn parse_diff_text(text: &str, cfg: &Config) -> Result, DiffError> { diff --git a/provn-cli/src/scanner/ast.rs b/provn-cli/src/scanner/ast.rs index 37fea81..fe1bbae 100644 --- a/provn-cli/src/scanner/ast.rs +++ b/provn-cli/src/scanner/ast.rs @@ -8,60 +8,61 @@ pub struct AstMatch { pub confidence: f64, } -pub fn scan_source(source: &str, lang: &str, cfg: &AstConfig) -> Option { +/// Scan `source` for sensitive variable assignments and return **all** matches. +/// Previously returned only the first match found during the tree walk, so a file +/// with both `system_prompt` and `api_key` assignments would silently miss the second. +pub fn scan_source(source: &str, lang: &str, cfg: &AstConfig) -> Vec { let language = match lang { "python" => tree_sitter_python::LANGUAGE.into(), "javascript" | "typescript" => tree_sitter_javascript::LANGUAGE.into(), - _ => return None, + _ => return vec![], }; let mut parser = Parser::new(); - parser.set_language(&language).ok()?; + if parser.set_language(&language).is_err() { + return vec![]; + } - let tree = parser.parse(source, None)?; - let root = tree.root_node(); + let tree = match parser.parse(source, None) { + Some(t) => t, + None => return vec![], + }; - scan_node(root, source.as_bytes(), cfg) + let mut matches = Vec::new(); + scan_node(tree.root_node(), source.as_bytes(), cfg, &mut matches); + matches } -fn scan_node(node: Node<'_>, src: &[u8], cfg: &AstConfig) -> Option { - // Look for assignment nodes - let node_kind = node.kind(); - if node_kind == "assignment" || node_kind == "expression_statement" { +fn scan_node(node: Node<'_>, src: &[u8], cfg: &AstConfig, out: &mut Vec) { + let kind = node.kind(); + if kind == "assignment" || kind == "expression_statement" { if let Some(m) = check_assignment(node, src, cfg) { - return Some(m); + out.push(m); } } - // Recurse into children let mut cursor = node.walk(); for child in node.children(&mut cursor) { - if let Some(m) = scan_node(child, src, cfg) { - return Some(m); - } + scan_node(child, src, cfg, out); } - None } fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option { - // Find identifier on left side and string/value on right let mut cursor = node.walk(); let children: Vec = node.children(&mut cursor).collect(); - // Try to find LHS identifier let lhs = children.iter().find(|n| n.kind() == "identifier")?; let lhs_text = lhs.utf8_text(src).ok()?; - // Check if this variable name is sensitive - let is_sensitive = cfg.sensitive_vars.iter().any(|v| { - lhs_text.to_lowercase().contains(v.as_str()) - }); + let is_sensitive = cfg + .sensitive_vars + .iter() + .any(|v| lhs_text.to_lowercase().contains(v.as_str())); if !is_sensitive { return None; } - // Find RHS string literal let rhs = children.iter().find(|n| { matches!( n.kind(), @@ -70,14 +71,12 @@ fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option })?; let rhs_text = rhs.utf8_text(src).ok()?; - - // Only flag if the string is substantial (not empty or trivial test values) let inner = rhs_text.trim_matches(|c| c == '"' || c == '\'' || c == '`'); + if inner.len() < 10 { return None; } - // Skip obvious test/placeholder values if inner.starts_with("test_") || inner.starts_with("fake_") || inner.starts_with("placeholder") @@ -90,8 +89,14 @@ fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option let line = lhs.start_position().row + 1; let snippet = node.utf8_text(src).ok()?.chars().take(100).collect(); - // Higher confidence for longer strings (more likely real secrets) - let confidence = if inner.len() > 50 { 0.85 } else { 0.70 }; + // Factor entropy of the RHS value into the confidence score instead of + // using string length alone — a long but low-entropy string is less suspicious. + let entropy = shannon_entropy(inner); + let confidence = if entropy >= 4.0 { + if inner.len() > 50 { 0.90 } else { 0.80 } + } else { + if inner.len() > 50 { 0.75 } else { 0.60 } + }; Some(AstMatch { var_name: lhs_text.to_string(), @@ -101,6 +106,25 @@ fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option }) } +fn shannon_entropy(s: &str) -> f64 { + if s.is_empty() { + return 0.0; + } + let len = s.len() as f64; + let mut counts = [0u32; 256]; + for b in s.bytes() { + counts[b as usize] += 1; + } + counts + .iter() + .filter(|&&c| c > 0) + .map(|&c| { + let p = c as f64 / len; + -p * p.log2() + }) + .sum() +} + #[cfg(test)] mod tests { use super::*; @@ -120,25 +144,30 @@ mod tests { #[test] fn detects_system_prompt_assignment() { - let src = r#"system_prompt = "You are a financial advisor with proprietary scoring.""#; - let result = scan_source(src, "python", &default_cfg()); - assert!(result.is_some()); - assert_eq!(result.unwrap().var_name, "system_prompt"); + let src = r#"system_prompt = "You are a financial advisor with proprietary scoring.""#; // provn:allow + let results = scan_source(src, "python", &default_cfg()); + assert!(!results.is_empty()); + assert_eq!(results[0].var_name, "system_prompt"); + } + + #[test] + fn detects_all_sensitive_assignments() { + let src = "system_prompt = \"You are a secret agent with classified intel.\"\napi_key = \"sk-proj-abcdefghijklmnopqrstuvwxyz123456\""; // provn:allow + let results = scan_source(src, "python", &default_cfg()); + assert_eq!(results.len(), 2, "expected both assignments to be caught"); } #[test] fn skips_test_values() { - let src = r#"api_key = "test_key_placeholder""#; + let src = r#"api_key = "test_key_placeholder""#; // provn:allow let result = scan_source(src, "python", &default_cfg()); - // "test_key_placeholder" starts with "test" and would be skipped… but "test_key_placeholder" doesn't start with "test_" - // This verifies the scanner runs without panic - let _ = result; + let _ = result; // verifies no panic; value may or may not be flagged } #[test] fn allows_non_sensitive_vars() { let src = r#"greeting = "Hello, World! This is a long enough string to trigger length checks.""#; - let result = scan_source(src, "python", &default_cfg()); - assert!(result.is_none()); + let results = scan_source(src, "python", &default_cfg()); + assert!(results.is_empty()); } } diff --git a/provn-cli/src/scanner/entropy.rs b/provn-cli/src/scanner/entropy.rs index 30e5d31..51fe9d3 100644 --- a/provn-cli/src/scanner/entropy.rs +++ b/provn-cli/src/scanner/entropy.rs @@ -21,7 +21,6 @@ pub fn scan_line(line: &str, cfg: &EntropyConfig) -> Option { for token in tokens { let h = shannon_entropy(token); if h >= cfg.threshold { - // Filter out obvious false positives if is_likely_false_positive(token) { continue; } @@ -55,21 +54,56 @@ fn shannon_entropy(s: &str) -> f64 { } fn is_likely_false_positive(token: &str) -> bool { - // Base64-encoded PNG/image headers are not secrets + // Base64-encoded image headers if token.starts_with("iVBORw0KGgo") || token.starts_with("/9j/") { return true; } - // Long hex strings that are hashes (64 chars = SHA-256) - if token.len() == 64 && token.chars().all(|c| c.is_ascii_hexdigit()) { + + // Pure hex strings at common hash lengths: MD5=32, SHA1=40, SHA256=64, SHA512=128 + if matches!(token.len(), 32 | 40 | 64 | 128) + && token.chars().all(|c| c.is_ascii_hexdigit()) + { + return true; + } + + // UUID: 8-4-4-4-12 hex groups separated by dashes + if is_uuid(token) { + return true; + } + + // npm/yarn/pip integrity hashes (e.g. "sha512-abc123==") + if token.starts_with("sha512-") || token.starts_with("sha256-") || token.starts_with("sha1-") { return true; } - // Looks like a URL path + + // Semver / dotted version strings with build metadata (1.2.3-rc.1+build.42) + // Three or more dot-separated segments means it's almost certainly a version, not a secret. + if token.chars().filter(|&c| c == '.').count() >= 2 + && token.chars().all(|c| c.is_ascii_alphanumeric() || ".-+_".contains(c)) + { + return true; + } + + // URLs and absolute paths if token.starts_with("http") || token.starts_with('/') { return true; } + false } +fn is_uuid(s: &str) -> bool { + let parts: Vec<&str> = s.split('-').collect(); + if parts.len() != 5 { + return false; + } + let expected_lens = [8usize, 4, 4, 4, 12]; + parts + .iter() + .zip(expected_lens.iter()) + .all(|(p, &len)| p.len() == len && p.chars().all(|c| c.is_ascii_hexdigit())) +} + #[cfg(test)] mod tests { use super::*; @@ -84,19 +118,38 @@ mod tests { #[test] fn flags_high_entropy_secret() { - let line = r#"secret = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn""#; + let line = r#"secret = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn""#; // provn:allow assert!(scan_line(line, &default_cfg()).is_some()); } #[test] fn skips_png_base64() { - let line = r#"icon = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk""#; + let line = r#"icon = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk""#; // provn:allow assert!(scan_line(line, &default_cfg()).is_none()); } #[test] fn skips_non_assignment_lines() { - let line = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn"; + let line = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn"; // provn:allow + assert!(scan_line(line, &default_cfg()).is_none()); + } + + #[test] + fn skips_uuid() { + let line = r#"trace_id = "550e8400-e29b-41d4-a716-446655440000""#; + assert!(scan_line(line, &default_cfg()).is_none()); + } + + #[test] + fn skips_sha256_hex() { + let hash = "a".repeat(64); // 64 hex chars + let line = format!(r#"checksum = "{hash}""#); + assert!(scan_line(&line, &default_cfg()).is_none()); + } + + #[test] + fn skips_npm_integrity_hash() { + let line = r#"integrity: "sha512-abc123def456ghi789jklmno012pqrstuvwxyz34567890ABCDEFGHIJKLMNOPQRSTUV===""#; assert!(scan_line(line, &default_cfg()).is_none()); } } diff --git a/provn-cli/src/scanner/mod.rs b/provn-cli/src/scanner/mod.rs index cb053b4..163fd40 100644 --- a/provn-cli/src/scanner/mod.rs +++ b/provn-cli/src/scanner/mod.rs @@ -21,20 +21,28 @@ pub struct ScanResult { pub latency_ms: u64, } +/// Maximum number of ambiguous candidates forwarded to Layer 3. +/// Capping at 3 bounds worst-case pre-commit latency while still catching +/// multi-secret diffs that the old single-candidate approach would miss. +const MAX_AMBIGUOUS: usize = 3; + /// Scan all chunks and return every finding, sorted by confidence descending. -/// Layer 3 (semantic) is invoked at most once — on the best ambiguous candidate — -/// to keep pre-commit latency low. pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { let start = Instant::now(); - let mut confirmed: Vec = Vec::new(); // high-confidence, no L3 needed - let mut ambiguous: Option = None; // best L1/L2 in the grey zone + // Compile user-defined patterns once before the scan loop to avoid + // recompiling the same regex on every line of the diff. + let compiled_custom = regex_scan::compile_custom_patterns(&cfg.layers.regex.custom_patterns); + + let mut confirmed: Vec = Vec::new(); + let mut ambiguous: Vec = Vec::new(); // top-N by confidence for chunk in chunks { for (line_num, line_content) in &chunk.added_lines { - // ── Layer 1a: regex ───────────────────────────────────────────── + // ── Layer 1a: regex — all matches on this line ─────────────────── if cfg.layers.regex.enabled { - if let Some(m) = regex_scan::scan_line(line_content, &cfg.layers.regex.custom_patterns) { + for m in regex_scan::scan_line(line_content, &compiled_custom) { + let conf = m.confidence; let r = ScanResult { file: Some(chunk.file.to_string_lossy().into_owned()), line: Some(*line_num), @@ -46,15 +54,15 @@ pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { ), snippet: Some(line_content.chars().take(120).collect()), redacted: Some(m.redacted), - confidence: m.confidence, + confidence: conf, layer: Some("regex".to_string()), - tier: Some(m.tier.clone()), + tier: Some(m.tier), latency_ms: 0, }; - if m.confidence >= cfg.layers.semantic.ambiguous_high { + if conf >= cfg.layers.semantic.ambiguous_high { confirmed.push(r); } else { - keep_best(&mut ambiguous, r); + add_ambiguous(&mut ambiguous, r); } } } @@ -62,6 +70,7 @@ pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { // ── Layer 1b: entropy ──────────────────────────────────────────── if cfg.layers.entropy.enabled { if let Some(m) = entropy::scan_line(line_content, &cfg.layers.entropy) { + let conf = m.confidence; let r = ScanResult { file: Some(chunk.file.to_string_lossy().into_owned()), line: Some(*line_num), @@ -69,21 +78,21 @@ pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { description: Some(format!("High entropy token (H={:.2})", m.entropy)), snippet: Some(line_content.chars().take(120).collect()), redacted: Some("PROVN_REDACTED_HIGH_ENTROPY".to_string()), - confidence: m.confidence, + confidence: conf, layer: Some("entropy".to_string()), tier: Some("T2".to_string()), latency_ms: 0, }; - if m.confidence >= cfg.layers.semantic.ambiguous_high { + if conf >= cfg.layers.semantic.ambiguous_high { confirmed.push(r); } else { - keep_best(&mut ambiguous, r); + add_ambiguous(&mut ambiguous, r); } } } } - // ── Layer 2: AST (per file) ────────────────────────────────────────── + // ── Layer 2: AST — all sensitive assignments in this file ──────────── if cfg.layers.ast.enabled { let src: String = chunk .added_lines @@ -93,13 +102,14 @@ pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { .join("\n"); let lang = match chunk.extension.as_str() { - "py" => Some("python"), - "ts" | "tsx" | "js" | "jsx" | "mjs" => Some("javascript"), - _ => None, + "py" => Some("python"), + "ts" | "tsx" | "js" | "jsx" | "mjs" => Some("javascript"), + _ => None, }; if let Some(lang) = lang { - if let Some(m) = ast::scan_source(&src, lang, &cfg.layers.ast) { + for m in ast::scan_source(&src, lang, &cfg.layers.ast) { + let conf = m.confidence; let r = ScanResult { file: Some(chunk.file.to_string_lossy().into_owned()), line: Some(m.line), @@ -110,53 +120,82 @@ pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { )), snippet: Some(m.snippet.chars().take(120).collect()), redacted: Some(format!("PROVN_REDACTED_{}", m.var_name.to_uppercase())), - confidence: m.confidence, + confidence: conf, layer: Some("ast".to_string()), tier: Some("T1".to_string()), latency_ms: 0, }; - if m.confidence >= cfg.layers.semantic.ambiguous_high { + if conf >= cfg.layers.semantic.ambiguous_high { confirmed.push(r); } else { - keep_best(&mut ambiguous, r); + add_ambiguous(&mut ambiguous, r); } } } } } - // ── Layer 3: semantic — one call on the best ambiguous candidate ────────── - if let Some(cand) = ambiguous { - let conf = cand.confidence; + // ── Layer 3: semantic — fan out to all ambiguous candidates ────────────── + if !ambiguous.is_empty() { let sem_cfg = &cfg.layers.semantic; - let ready = sem_cfg.enabled && !sem_cfg.model.trim().is_empty(); - - if ready && conf >= sem_cfg.ambiguous_low && conf < sem_cfg.ambiguous_high { - let code = cand.snippet.as_deref().unwrap_or(""); - let sem = semantic::classify(code, &sem_cfg.endpoint, sem_cfg.timeout_ms); - - if !sem.skipped { - if sem.label == "leak" { - let mut c = cand; - c.confidence = 0.85; - c.layer = Some("semantic".to_string()); - confirmed.push(c); + let ready = sem_cfg.enabled && !sem_cfg.model.trim().is_empty(); + + if ready { + let endpoint = sem_cfg.endpoint.clone(); + let timeout_ms = sem_cfg.timeout_ms; + let lo = sem_cfg.ambiguous_low; + let hi = sem_cfg.ambiguous_high; + let fallback = sem_cfg.fallback.clone(); + + let handles: Vec<_> = ambiguous + .into_iter() + .map(|cand| { + let ep = endpoint.clone(); + let fb = fallback.clone(); + std::thread::spawn(move || -> Option { + let conf = cand.confidence; + if conf >= lo && conf < hi { + let code = cand.snippet.as_deref().unwrap_or("").to_string(); + let sem = semantic::classify(&code, &ep, timeout_ms); + if sem.skipped { + if fb != "clean" { Some(cand) } else { None } + } else if sem.label == "leak" { + let mut c = cand; + c.confidence = 0.85; + c.layer = Some("semantic".to_string()); + Some(c) + } else { + None // L3 cleared it + } + } else if conf >= lo { + Some(cand) // above band — include as-is + } else { + None + } + }) + }) + .collect(); + + for handle in handles { + if let Ok(Some(r)) = handle.join() { + confirmed.push(r); } - // "clean" → L3 cleared it, discard - } else { - // L3 unavailable — apply fallback policy - if sem_cfg.fallback != "clean" { + } + } else { + // L3 not configured — include any candidate above the low threshold + for cand in ambiguous { + if cand.confidence >= sem_cfg.ambiguous_low { confirmed.push(cand); } } - } else if conf >= sem_cfg.ambiguous_low { - // Outside ambiguous band but above low threshold — include as-is - confirmed.push(cand); } } - // Sort by confidence descending and stamp shared latency - confirmed.sort_by(|a, b| b.confidence.partial_cmp(&a.confidence).unwrap_or(std::cmp::Ordering::Equal)); + confirmed.sort_by(|a, b| { + b.confidence + .partial_cmp(&a.confidence) + .unwrap_or(std::cmp::Ordering::Equal) + }); let elapsed = start.elapsed().as_millis() as u64; for r in &mut confirmed { r.latency_ms = elapsed; @@ -164,10 +203,21 @@ pub fn scan_chunks(chunks: &[DiffChunk], cfg: &Config) -> Vec { confirmed } -/// Keep the higher-confidence candidate in the ambiguous pool. -fn keep_best(slot: &mut Option, new: ScanResult) { - match slot { - Some(existing) if existing.confidence >= new.confidence => {} - _ => *slot = Some(new), +/// Keep the top [`MAX_AMBIGUOUS`] candidates by confidence. +fn add_ambiguous(pool: &mut Vec, new: ScanResult) { + if pool.len() < MAX_AMBIGUOUS { + pool.push(new); + pool.sort_by(|a, b| { + b.confidence + .partial_cmp(&a.confidence) + .unwrap_or(std::cmp::Ordering::Equal) + }); + } else if pool.last().map_or(false, |w| new.confidence > w.confidence) { + *pool.last_mut().unwrap() = new; + pool.sort_by(|a, b| { + b.confidence + .partial_cmp(&a.confidence) + .unwrap_or(std::cmp::Ordering::Equal) + }); } } diff --git a/provn-cli/src/scanner/regex_scan.rs b/provn-cli/src/scanner/regex_scan.rs index 74b707a..b085ec7 100644 --- a/provn-cli/src/scanner/regex_scan.rs +++ b/provn-cli/src/scanner/regex_scan.rs @@ -18,6 +18,16 @@ struct Pattern { redacted_prefix: &'static str, } +/// Pre-compiled form of a user-defined custom pattern. +/// Compile once via [`compile_custom_patterns`] before the scan loop. +pub struct CompiledCustomPattern { + pub name: String, + pub tier: String, + pub confidence: f64, + pub description: Option, + pub re: Regex, +} + static PATTERNS: Lazy> = Lazy::new(|| { vec![ Pattern { @@ -38,7 +48,7 @@ static PATTERNS: Lazy> = Lazy::new(|| { name: "openai_api_key", tier: "T1", confidence: 0.97, - re: Regex::new(r"sk-(?:proj-)?[a-zA-Z0-9]{40,}").unwrap(), + re: Regex::new(r"sk-(?:proj-|svcacct-)?[a-zA-Z0-9]{40,}").unwrap(), redacted_prefix: "PROVN_REDACTED_OPENAI_KEY", }, Pattern { @@ -121,38 +131,57 @@ static PATTERNS: Lazy> = Lazy::new(|| { ] }); -pub fn scan_line(line: &str, custom: &[crate::config::CustomPattern]) -> Option { +/// Compile user-defined patterns from config once before the scan loop. +/// Patterns that fail to compile are silently skipped. +pub fn compile_custom_patterns(patterns: &[crate::config::CustomPattern]) -> Vec { + patterns + .iter() + .filter_map(|cp| { + Regex::new(&cp.pattern).ok().map(|re| CompiledCustomPattern { + name: cp.name.clone(), + tier: cp.tier.clone(), + confidence: cp.confidence, + description: cp.description.clone(), + re, + }) + }) + .collect() +} + +/// Scan one line and return **all** matches, sorted by confidence descending. +/// Previously returned only the first match, so a line with both an AWS key and +/// an OpenAI key would silently drop one of them. +pub fn scan_line(line: &str, custom: &[CompiledCustomPattern]) -> Vec { // NFKC normalize to catch homoglyph attacks (Cyrillic 'а' → 'a') let normalized: String = line.nfkc().collect(); + let mut matches: Vec = Vec::new(); for pattern in PATTERNS.iter() { if pattern.re.is_match(&normalized) { - return Some(RegexMatch { + matches.push(RegexMatch { pattern_name: pattern.name.to_string(), - tier: pattern.tier.to_string(), - confidence: pattern.confidence, - redacted: format!("{}_1", pattern.redacted_prefix), - description: None, + tier: pattern.tier.to_string(), + confidence: pattern.confidence, + redacted: format!("{}_1", pattern.redacted_prefix), + description: None, }); } } - // User-defined patterns (from provn.yml regex.custom_patterns) for cp in custom { - if let Ok(re) = Regex::new(&cp.pattern) { - if re.is_match(&normalized) { - return Some(RegexMatch { - pattern_name: cp.name.clone(), - tier: cp.tier.clone(), - confidence: cp.confidence, - redacted: format!("PROVN_REDACTED_{}", cp.name.to_uppercase().replace(' ', "_")), - description: cp.description.clone(), - }); - } + if cp.re.is_match(&normalized) { + matches.push(RegexMatch { + pattern_name: cp.name.clone(), + tier: cp.tier.clone(), + confidence: cp.confidence, + redacted: format!("PROVN_REDACTED_{}", cp.name.to_uppercase().replace(' ', "_")), + description: cp.description.clone(), + }); } } - None + matches.sort_by(|a, b| b.confidence.partial_cmp(&a.confidence).unwrap_or(std::cmp::Ordering::Equal)); + matches } #[cfg(test)] @@ -161,42 +190,57 @@ mod tests { #[test] fn detects_aws_access_key() { - assert!(scan_line("AWS_ACCESS_KEY_ID = \"AKIAIOSFODNN7EXAMPLE\"", &[]).is_some()); + assert!(!scan_line("AWS_ACCESS_KEY_ID = \"AKIAIOSFODNN7EXAMPLE\"", &[]).is_empty()); // provn:allow } #[test] fn detects_openai_key() { - assert!(scan_line("key = \"sk-proj-abcdefghijklmnopqrstuvwxyz1234567890ABCD\"", &[]).is_some()); + assert!(!scan_line("key = \"sk-proj-abcdefghijklmnopqrstuvwxyz1234567890ABCD\"", &[]).is_empty()); // provn:allow } #[test] fn detects_private_key_header() { - assert!(scan_line("-----BEGIN RSA PRIVATE KEY-----", &[]).is_some()); + assert!(!scan_line("-----BEGIN RSA PRIVATE KEY-----", &[]).is_empty()); // provn:allow } #[test] fn allows_clean_code() { - assert!(scan_line("def calculate_total(items): return sum(items)", &[]).is_none()); + assert!(scan_line("def calculate_total(items): return sum(items)", &[]).is_empty()); } #[test] fn detects_homoglyph_aws_key() { // Cyrillic А (U+0410) instead of Latin A — NFKC normalizes it let homoglyph_line = "АKIАIOSFODNNsomething7EXАMPLE"; - // This may or may not match depending on normalization result let _ = scan_line(homoglyph_line, &[]); } + #[test] + fn returns_all_matches_on_multi_secret_line() { + // A line containing both an AWS key and an OpenAI key must report both. + let line = "AKIAIOSFODNN7EXAMPLE sk-proj-abcdefghijklmnopqrstuvwxyz1234567890ABCD"; // provn:allow + let hits = scan_line(line, &[]); + assert!(hits.len() >= 2, "expected ≥2 matches, got {}", hits.len()); + } + + #[test] + fn results_sorted_by_confidence_descending() { + let line = "AKIAIOSFODNN7EXAMPLE sk-proj-abcdefghijklmnopqrstuvwxyz1234567890ABCD"; // provn:allow + let hits = scan_line(line, &[]); + for w in hits.windows(2) { + assert!(w[0].confidence >= w[1].confidence); + } + } + #[test] fn detects_custom_pattern() { - use crate::config::CustomPattern; - let cp = CustomPattern { - name: "internal_import".to_string(), - pattern: r"from corp_internal\.".to_string(), - tier: "T1".to_string(), - confidence: 0.9, + let cp = CompiledCustomPattern { + name: "internal_import".to_string(), + tier: "T1".to_string(), + confidence: 0.9, description: None, + re: Regex::new(r"from corp_internal\.").unwrap(), }; - assert!(scan_line("from corp_internal.utils import helper", &[cp]).is_some()); + assert!(!scan_line("from corp_internal.utils import helper", &[cp]).is_empty()); } }