diff --git a/.github/dependabot.yml b/.github/dependabot.yml index cf398222..f1df2434 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,27 +10,12 @@ updates: update-types: ["minor", "patch"] # Direct bumps that can never resolve: a parent crate pins the old major and # its API type flows into our code. The real fix arrives when the PARENT - # ships a new release, so watch for a `scraper` / `pdf-inspector` bump PR and - # remove the matching ignore entry then. + # ships a new release, so watch for a `scraper` bump PR and remove the + # matching ignore entry then. ignore: # scraper 0.25 (latest) pins ego-tree 0.10; NodeId comes from scraper's # ElementRef, so a direct ego-tree 0.11 bump is a type mismatch. - dependency-name: "ego-tree" - # pdf-inspector 0.1.7 (latest, github.com/firecrawl/pdf-inspector) still - # pins lopdf 0.41; a direct bump leaves 0.41 in the tree anyway. lopdf - # 0.42 carries the RUSTSEC-2026-0187 fix (stack-overflow SIGABRT on - # deeply nested objects, uncatchable by catch_unwind). Contained for us: - # `document.sandbox` runs every parse in a child process and is ON in - # config.docker.toml, so an abort kills the worker, not the server. - # Drop this ignore once pdf-inspector publishes a crate release built on - # lopdf >= 0.42 (fixed on their main in 1c32e4bd, unreleased). - - dependency-name: "lopdf" - # 0.1.7 specifically: it extracts 13,732 fewer characters than 0.1.6 over - # our 30-PDF measurement corpus, one document losing six whole pages. The - # workspace pin holds 0.1.6; this keeps dependabot from re-proposing the - # bad release while still offering 0.1.8 when it lands. - - dependency-name: "pdf-inspector" - versions: ["0.1.7"] - package-ecosystem: "github-actions" directory: "/" diff --git a/Cargo.lock b/Cargo.lock index 83222da5..a9cd5853 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -835,7 +835,7 @@ dependencies = [ "anyhow", "assert_cmd", "axum", - "base64 0.22.1", + "base64 0.23.1", "clap", "console", "crw-browse", @@ -966,7 +966,7 @@ dependencies = [ name = "crw-mcp" version = "0.32.0" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "clap", "crw-core", "crw-renderer", @@ -1041,7 +1041,7 @@ version = "0.32.0" dependencies = [ "axum", "axum-test", - "base64 0.22.1", + "base64 0.23.1", "clap", "crw-core", "crw-crawl", @@ -2185,6 +2185,25 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -2385,9 +2404,9 @@ dependencies = [ [[package]] name = "jsonschema" -version = "0.50.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd2225bc56c338f18465ce618e5e56fff1295fcb0a0cb338b28010193f26949" +checksum = "29b6bb4e22283b09119c671e57ac6a185e9a0c70c31585e56b729b626d877753" dependencies = [ "ahash", "bytecount", @@ -2512,9 +2531,9 @@ dependencies = [ [[package]] name = "lopdf" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67513274c50a2b51e5f75d9e682fcf4ab064a8a9c9ae2c3c59309084882bb24d" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" dependencies = [ "aes", "bitflags 2.13.0", @@ -2919,11 +2938,12 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pdf-inspector" -version = "0.1.6" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5085fc777cc02a305e521c8a938ee662508fdd6c2aaca37bce4c08b3f527eed" +checksum = "6cdfc6057e1b38a2ae84490c5e64abc5c81738d4d5ac1ccc55cf1a2c9b87334e" dependencies = [ "env_logger", + "include_dir", "log", "lopdf", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 8316e9a7..5e6ab471 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,12 +63,13 @@ regex = "1" # libraries — preserves the single-static-binary story. The open-core # boundary gate asserts lopdf/ttf-parser are ABSENT when `pdf` is disabled. # -# Held at 0.1.6 on purpose. 0.1.7 guts six consecutive pages of some documents: -# measured over 30 real PDFs it extracts 13,732 fewer characters in total than -# 0.1.6, all of it one document (arXiv 1412.6980) losing pages 6-11 down to -# 4-13% of their text. 0.1.6 is the best release we have measured: +23,148 -# characters over 0.1.4 across the same corpus. Re-evaluate on 0.1.8. -pdf-inspector = "=0.1.6" +# Was held at 0.1.6 because 0.1.7 gutted pages 6-11 of arXiv 1412.6980. Measured +# again over 29 real PDFs (papers, IRS forms, RFCs, 10-K, the 756-page PDF spec) +# scoring word recall against poppler as ground truth: 1.17.0 reaches 98.15% vs +# 0.1.6's 97.72%, better on 23 of 29 documents, and every page of 1412.6980 now +# comes back at 97-100%. The raw character count drops slightly only because +# 1.17.0 rejoins hyphenated line breaks and stops duplicating table rows. +pdf-inspector = "=1.17.0" # Diffing (change-tracking / monitor). Myers diff over lines; the parse-diff # AST and the unified text surface are both derived from its op stream. @@ -112,7 +113,7 @@ uuid = { version = "1", features = ["v4", "serde"] } url = { version = "2", features = ["serde"] } sha2 = "0.11" hex = "0.4" -base64 = "0.22" +base64 = "0.23" # Unix process-group kill (browser teardown). Unix-only; already present # transitively. Used by crw-renderer's BROWSER_PGIDS group-kill registry. diff --git a/crates/crw-extract/Cargo.toml b/crates/crw-extract/Cargo.toml index ec517672..533e28c0 100644 --- a/crates/crw-extract/Cargo.toml +++ b/crates/crw-extract/Cargo.toml @@ -23,7 +23,7 @@ pdf-inspector = { workspace = true, optional = true } # Used ONLY for the pre-parse decompression-bomb guard: structure-parse the PDF # and bounded-inflate each FlateDecode stream so a malicious file is rejected # before pdf-inspector allocates the full (huge) decompressed payload. -lopdf = { version = "0.41", optional = true } +lopdf = { version = "0.42", optional = true } flate2 = { version = "1", optional = true } lol_html = { workspace = true } scraper = { workspace = true } diff --git a/crates/crw-extract/tests/pdf_tests.rs b/crates/crw-extract/tests/pdf_tests.rs index 8bd5e2cf..bafe97d7 100644 --- a/crates/crw-extract/tests/pdf_tests.rs +++ b/crates/crw-extract/tests/pdf_tests.rs @@ -74,3 +74,56 @@ fn error_codes_are_stable() { assert_eq!(PdfError::Corrupt("x".into()).code(), "pdf_parse_failed"); assert_eq!(PdfError::Disabled.code(), "pdf_disabled"); } + +/// RUSTSEC-2026-0187: lopdf <= 0.41 parsed nested arrays with unbounded +/// recursion, so a ~21 KB PDF whose Catalog holds a 10,000-deep array blew the +/// stack and aborted the process. A `SIGABRT` is not unwinding, so neither the +/// `catch_unwind` in `pdf::convert` nor this harness can trap it: on a +/// regression the whole test binary dies rather than reporting a failure, which +/// is the loudest signal available. Fixed in lopdf 0.42, which we reach through +/// pdf-inspector 1.17. +#[test] +fn deeply_nested_objects_do_not_abort() { + let depth = 10_380; + let nested: Vec = b"[" + .repeat(depth) + .into_iter() + .chain(b"]".repeat(depth)) + .collect(); + + let objects: Vec> = vec![ + [ + b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R /X ".as_slice(), + &nested, + b" >>\nendobj\n", + ] + .concat(), + b"2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n".to_vec(), + b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n".to_vec(), + ]; + + let mut pdf = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for object in &objects { + offsets.push(pdf.len()); + pdf.extend_from_slice(object); + } + let startxref = pdf.len(); + pdf.extend_from_slice( + format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes(), + ); + for offset in &offsets { + pdf.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice( + format!( + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{startxref}\n%%EOF\n", + objects.len() + 1 + ) + .as_bytes(), + ); + + // Reaching the assertion at all is the point: extracting nothing from a + // page-less document is fine, aborting the process is not. + let _ = pdf::convert(&pdf, false, None, 5 * 1024 * 1024); +}