fix(scanner): 5 correctness and FP improvements

provn-cli/src/diff.rs

@@ -32,30 +32,39 @@ pub fn parse_staged_diff(cfg: &Config) -> Result<Vec<DiffChunk>, DiffError> {
 }
 
 pub fn parse_file(path: &str, cfg: &Config) -> Result<Vec<DiffChunk>, DiffError> {
+    let file_path = PathBuf::from(path);
+
+    if should_skip(&file_path, cfg) {
+        return Ok(vec![]);
+    }
+
     let content = std::fs::read_to_string(path)?;
-    let ext = PathBuf::from(path)
+
+    // Honour provn:skip-file anywhere in the file — same semantics as in diff mode.
+    if content.contains("provn:skip-file") || content.contains("aegis:skip-file") {
+        return Ok(vec![]);
+    }
+
+    let ext = file_path
         .extension()
         .map(|e| e.to_string_lossy().to_string())
         .unwrap_or_default();
 
-    // Synthesize fake added lines (all lines are "added" when checking a file directly)
+    // Synthesize "added" lines from every line in the file, applying the same
+    // provn:allow filter that parse_diff_text uses so the annotation works in
+    // both `provn scan` (pre-commit) and `provn check` (file) modes.
     let added_lines: Vec<(usize, String)> = content
         .lines()
         .enumerate()
+        .filter(|(_, l)| !l.contains("provn:allow") && !l.contains("aegis:allow"))
         .map(|(i, l)| (i + 1, l.to_string()))
         .collect();
 
-    let chunk = DiffChunk {
-        file: PathBuf::from(path),
+    Ok(vec![DiffChunk {
+        file: file_path,
         extension: ext,
         added_lines,
-    };
-
-    if should_skip(&chunk.file, cfg) {
-        return Ok(vec![]);
-    }
-
-    Ok(vec![chunk])
+    }])
 }
 
 fn parse_diff_text(text: &str, cfg: &Config) -> Result<Vec<DiffChunk>, DiffError> {

provn-cli/src/scanner/ast.rs

@@ -8,60 +8,61 @@ pub struct AstMatch {
     pub confidence: f64,
 }
 
-pub fn scan_source(source: &str, lang: &str, cfg: &AstConfig) -> Option<AstMatch> {
+/// Scan `source` for sensitive variable assignments and return **all** matches.
+/// Previously returned only the first match found during the tree walk, so a file
+/// with both `system_prompt` and `api_key` assignments would silently miss the second.
+pub fn scan_source(source: &str, lang: &str, cfg: &AstConfig) -> Vec<AstMatch> {
     let language = match lang {
         "python" => tree_sitter_python::LANGUAGE.into(),
         "javascript" | "typescript" => tree_sitter_javascript::LANGUAGE.into(),
-        _ => return None,
+        _ => return vec![],
     };
 
     let mut parser = Parser::new();
-    parser.set_language(&language).ok()?;
+    if parser.set_language(&language).is_err() {
+        return vec![];
+    }
 
-    let tree = parser.parse(source, None)?;
-    let root = tree.root_node();
+    let tree = match parser.parse(source, None) {
+        Some(t) => t,
+        None => return vec![],
+    };
 
-    scan_node(root, source.as_bytes(), cfg)
+    let mut matches = Vec::new();
+    scan_node(tree.root_node(), source.as_bytes(), cfg, &mut matches);
+    matches
 }
 
-fn scan_node(node: Node<'_>, src: &[u8], cfg: &AstConfig) -> Option<AstMatch> {
-    // Look for assignment nodes
-    let node_kind = node.kind();
-    if node_kind == "assignment" || node_kind == "expression_statement" {
+fn scan_node(node: Node<'_>, src: &[u8], cfg: &AstConfig, out: &mut Vec<AstMatch>) {
+    let kind = node.kind();
+    if kind == "assignment" || kind == "expression_statement" {
         if let Some(m) = check_assignment(node, src, cfg) {
-            return Some(m);
+            out.push(m);
         }
     }
 
-    // Recurse into children
     let mut cursor = node.walk();
     for child in node.children(&mut cursor) {
-        if let Some(m) = scan_node(child, src, cfg) {
-            return Some(m);
-        }
+        scan_node(child, src, cfg, out);
     }
-    None
 }
 
 fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option<AstMatch> {
-    // Find identifier on left side and string/value on right
     let mut cursor = node.walk();
     let children: Vec<Node> = node.children(&mut cursor).collect();
 
-    // Try to find LHS identifier
     let lhs = children.iter().find(|n| n.kind() == "identifier")?;
     let lhs_text = lhs.utf8_text(src).ok()?;
 
-    // Check if this variable name is sensitive
-    let is_sensitive = cfg.sensitive_vars.iter().any(|v| {
-        lhs_text.to_lowercase().contains(v.as_str())
-    });
+    let is_sensitive = cfg
+        .sensitive_vars
+        .iter()
+        .any(|v| lhs_text.to_lowercase().contains(v.as_str()));
 
     if !is_sensitive {
         return None;
     }
 
-    // Find RHS string literal
     let rhs = children.iter().find(|n| {
         matches!(
             n.kind(),
@@ -70,14 +71,12 @@ fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option<AstMatch>
     })?;
 
     let rhs_text = rhs.utf8_text(src).ok()?;
-
-    // Only flag if the string is substantial (not empty or trivial test values)
     let inner = rhs_text.trim_matches(|c| c == '"' || c == '\'' || c == '`');
+
     if inner.len() < 10 {
         return None;
     }
 
-    // Skip obvious test/placeholder values
     if inner.starts_with("test_")
         || inner.starts_with("fake_")
         || inner.starts_with("placeholder")
@@ -90,8 +89,14 @@ fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option<AstMatch>
     let line = lhs.start_position().row + 1;
     let snippet = node.utf8_text(src).ok()?.chars().take(100).collect();
 
-    // Higher confidence for longer strings (more likely real secrets)
-    let confidence = if inner.len() > 50 { 0.85 } else { 0.70 };
+    // Factor entropy of the RHS value into the confidence score instead of
+    // using string length alone — a long but low-entropy string is less suspicious.
+    let entropy = shannon_entropy(inner);
+    let confidence = if entropy >= 4.0 {
+        if inner.len() > 50 { 0.90 } else { 0.80 }
+    } else {
+        if inner.len() > 50 { 0.75 } else { 0.60 }
+    };
 
     Some(AstMatch {
         var_name: lhs_text.to_string(),
@@ -101,6 +106,25 @@ fn check_assignment(node: Node, src: &[u8], cfg: &AstConfig) -> Option<AstMatch>
     })
 }
 
+fn shannon_entropy(s: &str) -> f64 {
+    if s.is_empty() {
+        return 0.0;
+    }
+    let len = s.len() as f64;
+    let mut counts = [0u32; 256];
+    for b in s.bytes() {
+        counts[b as usize] += 1;
+    }
+    counts
+        .iter()
+        .filter(|&&c| c > 0)
+        .map(|&c| {
+            let p = c as f64 / len;
+            -p * p.log2()
+        })
+        .sum()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -120,25 +144,30 @@ mod tests {
 
     #[test]
     fn detects_system_prompt_assignment() {
-        let src = r#"system_prompt = "You are a financial advisor with proprietary scoring.""#;
-        let result = scan_source(src, "python", &default_cfg());
-        assert!(result.is_some());
-        assert_eq!(result.unwrap().var_name, "system_prompt");
+        let src = r#"system_prompt = "You are a financial advisor with proprietary scoring.""#; // provn:allow
+        let results = scan_source(src, "python", &default_cfg());
+        assert!(!results.is_empty());
+        assert_eq!(results[0].var_name, "system_prompt");
+    }
+
+    #[test]
+    fn detects_all_sensitive_assignments() {
+        let src = "system_prompt = \"You are a secret agent with classified intel.\"\napi_key = \"sk-proj-abcdefghijklmnopqrstuvwxyz123456\""; // provn:allow
+        let results = scan_source(src, "python", &default_cfg());
+        assert_eq!(results.len(), 2, "expected both assignments to be caught");
     }
 
     #[test]
     fn skips_test_values() {
-        let src = r#"api_key = "test_key_placeholder""#;
+        let src = r#"api_key = "test_key_placeholder""#; // provn:allow
         let result = scan_source(src, "python", &default_cfg());
-        // "test_key_placeholder" starts with "test" and would be skipped… but "test_key_placeholder" doesn't start with "test_"
-        // This verifies the scanner runs without panic
-        let _ = result;
+        let _ = result; // verifies no panic; value may or may not be flagged
     }
 
     #[test]
     fn allows_non_sensitive_vars() {
         let src = r#"greeting = "Hello, World! This is a long enough string to trigger length checks.""#;
-        let result = scan_source(src, "python", &default_cfg());
-        assert!(result.is_none());
+        let results = scan_source(src, "python", &default_cfg());
+        assert!(results.is_empty());
     }
 }

provn-cli/src/scanner/entropy.rs

@@ -21,7 +21,6 @@ pub fn scan_line(line: &str, cfg: &EntropyConfig) -> Option<EntropyMatch> {
     for token in tokens {
         let h = shannon_entropy(token);
         if h >= cfg.threshold {
-            // Filter out obvious false positives
             if is_likely_false_positive(token) {
                 continue;
             }
@@ -55,21 +54,56 @@ fn shannon_entropy(s: &str) -> f64 {
 }
 
 fn is_likely_false_positive(token: &str) -> bool {
-    // Base64-encoded PNG/image headers are not secrets
+    // Base64-encoded image headers
     if token.starts_with("iVBORw0KGgo") || token.starts_with("/9j/") {
         return true;
     }
-    // Long hex strings that are hashes (64 chars = SHA-256)
-    if token.len() == 64 && token.chars().all(|c| c.is_ascii_hexdigit()) {
+
+    // Pure hex strings at common hash lengths: MD5=32, SHA1=40, SHA256=64, SHA512=128
+    if matches!(token.len(), 32 | 40 | 64 | 128)
+        && token.chars().all(|c| c.is_ascii_hexdigit())
+    {
+        return true;
+    }
+
+    // UUID: 8-4-4-4-12 hex groups separated by dashes
+    if is_uuid(token) {
+        return true;
+    }
+
+    // npm/yarn/pip integrity hashes (e.g. "sha512-abc123==")
+    if token.starts_with("sha512-") || token.starts_with("sha256-") || token.starts_with("sha1-") {
         return true;
     }
-    // Looks like a URL path
+
+    // Semver / dotted version strings with build metadata (1.2.3-rc.1+build.42)
+    // Three or more dot-separated segments means it's almost certainly a version, not a secret.
+    if token.chars().filter(|&c| c == '.').count() >= 2
+        && token.chars().all(|c| c.is_ascii_alphanumeric() || ".-+_".contains(c))
+    {
+        return true;
+    }
+
+    // URLs and absolute paths
     if token.starts_with("http") || token.starts_with('/') {
         return true;
     }
+
     false
 }
 
+fn is_uuid(s: &str) -> bool {
+    let parts: Vec<&str> = s.split('-').collect();
+    if parts.len() != 5 {
+        return false;
+    }
+    let expected_lens = [8usize, 4, 4, 4, 12];
+    parts
+        .iter()
+        .zip(expected_lens.iter())
+        .all(|(p, &len)| p.len() == len && p.chars().all(|c| c.is_ascii_hexdigit()))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -84,19 +118,38 @@ mod tests {
 
     #[test]
     fn flags_high_entropy_secret() {
-        let line = r#"secret = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn""#;
+        let line = r#"secret = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn""#; // provn:allow
         assert!(scan_line(line, &default_cfg()).is_some());
     }
 
     #[test]
     fn skips_png_base64() {
-        let line = r#"icon = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk""#;
+        let line = r#"icon = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk""#; // provn:allow
         assert!(scan_line(line, &default_cfg()).is_none());
     }
 
     #[test]
     fn skips_non_assignment_lines() {
-        let line = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn";
+        let line = "x7Kp2mNqR9vT4wYjLhBcDfAeGiUoSzXn"; // provn:allow
+        assert!(scan_line(line, &default_cfg()).is_none());
+    }
+
+    #[test]
+    fn skips_uuid() {
+        let line = r#"trace_id = "550e8400-e29b-41d4-a716-446655440000""#;
+        assert!(scan_line(line, &default_cfg()).is_none());
+    }
+
+    #[test]
+    fn skips_sha256_hex() {
+        let hash = "a".repeat(64); // 64 hex chars
+        let line = format!(r#"checksum = "{hash}""#);
+        assert!(scan_line(&line, &default_cfg()).is_none());
+    }
+
+    #[test]
+    fn skips_npm_integrity_hash() {
+        let line = r#"integrity: "sha512-abc123def456ghi789jklmno012pqrstuvwxyz34567890ABCDEFGHIJKLMNOPQRSTUV===""#;
         assert!(scan_line(line, &default_cfg()).is_none());
     }
 }