From d61fa45b335947bc96d83db2f6fe35b9c65b2167 Mon Sep 17 00:00:00 2001 From: rqi14 <26152437+rqi14@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:12:35 +0100 Subject: [PATCH 1/3] feat(renderer): let camoufox wait out a JS challenge instead of reporting it `run_sequence_inner` evaluates `outerHTML` exactly once and immediately judges the result (`crates/crw-renderer/src/camoufox.rs:230-247`). A Cloudflare-style JS challenge resolves client-side several seconds after navigation, so that single evaluate can only ever observe the interstitial. The tier then reports a retryable wall for pages it would have gotten had it looked again -- and since camoufox is normally the last tier, that failure is the request's failure. The CDP tiers already solve this. `cdp.rs` polls a challenge through `CHALLENGE_MAX_RETRIES` / `CHALLENGE_POLL_INTERVAL_MS`, sizes the reservation with `challenge_retry_budget()`, and exposes the knob as `chrome_challenge_max_retries` wired in via `.with_challenge_retries(...)` at both CDP construction sites. Waiting out a challenge is a first-class concept in this codebase; camoufox is the one tier that never got it. `renderer.camoufox_challenge_wait_ms` (default 20s, `0` restores today's single-shot behaviour) gives camoufox the same loop, configured the same way -- `CamoufoxRenderer::new(...).with_challenge_wait(...)`, mirroring `CdpRenderer::new(...).with_challenge_retries(...)`. `new` keeps its four arguments, so this is additive for any external caller. The loop polls only `looks_like_wall() == Some("challenge")`. The `"wall"` markers ("attention required! | cloudflare", "enable javascript and cookies to continue") are terminal refusals, not work in progress; polling those would spend the entire ceiling to arrive at the identical error. A clean page never enters the loop at all. Env: `CRW_RENDERER__CAMOUFOX_CHALLENGE_WAIT_MS`. --- config.default.toml | 1 + crates/crw-core/src/config.rs | 45 +++++++++ crates/crw-renderer/src/camoufox.rs | 142 +++++++++++++++++++++++++++- crates/crw-renderer/src/lib.rs | 15 +-- docs/docs/js-rendering.md | 4 + 5 files changed, 199 insertions(+), 8 deletions(-) diff --git a/config.default.toml b/config.default.toml index b659c438..e5e22870 100644 --- a/config.default.toml +++ b/config.default.toml @@ -66,6 +66,7 @@ ws_url = "ws://127.0.0.1:9222/" # api_key = "" # optional bearer token if the sidecar is protected # include_in_auto = false # default: stay out of the auto ladder # camoufox_timeout_ms = 60000 # optional per-request REST budget override +# camoufox_challenge_wait_ms = 20000 # ceiling for polling out a CF-style JS challenge (0 disables) [crawler] max_concurrency = 10 diff --git a/crates/crw-core/src/config.rs b/crates/crw-core/src/config.rs index 63fc6670..db6003eb 100644 --- a/crates/crw-core/src/config.rs +++ b/crates/crw-core/src/config.rs @@ -221,6 +221,15 @@ pub const MAX_WAIT_FOR_MS: u64 = 60_000; /// and must never be charged CDP overhead. pub const CAMOUFOX_DEFAULT_TIMEOUT_MS: u64 = 60_000; +/// Default ceiling (ms) for polling a Camoufox tab while a Cloudflare-style JS +/// challenge clears itself, before giving up and reporting the wall. The CDP +/// tiers already run this loop ([`crw-renderer`'s `CHALLENGE_MAX_RETRIES`] × +/// `CHALLENGE_POLL_INTERVAL_MS` = 9s); Camoufox is a slower engine reached +/// after the CDP tiers have already failed, so it gets a wider ceiling. 20s +/// covers the commonly observed 5-25s clear without eating most of a 60s +/// request budget. Used by [`RendererConfig::camoufox_challenge_wait`]. +pub const CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS: u64 = 20_000; + /// Default cloak (Turnstile-solver sidecar) per-request budget (ms) — the /// per-attempt solve budget bounding one sidecar mirror call. A cold interactive /// Turnstile solve is ~21s; 35s leaves margin for the browser solve + curl_cffi @@ -803,6 +812,14 @@ pub struct RendererConfig { /// [`CAMOUFOX_DEFAULT_TIMEOUT_MS`] when unset. #[serde(default)] pub camoufox_timeout_ms: Option, + /// Ceiling (ms) for polling a Camoufox tab while a bot-challenge + /// interstitial clears, before giving up and reporting the wall. Falls back + /// to [`CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS`] when unset; `Some(0)` disables + /// the poll and restores the single-shot behaviour. The Camoufox analogue + /// of `chrome_challenge_max_retries`. + /// Env: `CRW_RENDERER__CAMOUFOX_CHALLENGE_WAIT_MS`. + #[serde(default)] + pub camoufox_challenge_wait_ms: Option, /// Opt-in cloak Turnstile-solver sidecar endpoint. See [`CloakEndpoint`]. /// `None` = not configured (default) → the tier is never constructed and the /// engine is byte-identical to a build without it. Fired only as a @@ -1080,6 +1097,7 @@ impl Default for RendererConfig { chrome_proxy_timeout_ms: None, camoufox: None, camoufox_timeout_ms: None, + camoufox_challenge_wait_ms: None, cloak: None, cloak_timeout_ms: None, cloak_proxy_host: None, @@ -1133,6 +1151,12 @@ impl RendererConfig { self.camoufox_timeout_ms .unwrap_or(CAMOUFOX_DEFAULT_TIMEOUT_MS) } + /// Camoufox bot-challenge poll ceiling (ms). Unconditional (no `#[cfg]`), + /// like [`Self::camoufox_timeout`]. + pub fn camoufox_challenge_wait(&self) -> u64 { + self.camoufox_challenge_wait_ms + .unwrap_or(CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS) + } /// Per-attempt cloak solve budget (ms). Unconditional (no `#[cfg]`) so /// `tier_timeouts_from` can reference it in every build, like /// [`Self::camoufox_timeout`]. @@ -3171,6 +3195,7 @@ search_backend_url = "http://from-file:8080" assert_eq!(r.chrome_proxy_timeout_ms, None); assert!(r.camoufox.is_none()); assert_eq!(r.camoufox_timeout_ms, None); + assert_eq!(r.camoufox_challenge_wait_ms, None); assert!(r.cloak.is_none()); assert_eq!(r.cloak_timeout_ms, None); assert_eq!(r.cloak_proxy_host, None); @@ -3355,6 +3380,26 @@ search_backend_url = "http://from-file:8080" assert_eq!(r2.camoufox_timeout(), 1_234); } + #[test] + fn camoufox_challenge_wait_default_and_override() { + let r = RendererConfig::default(); + assert_eq!( + r.camoufox_challenge_wait(), + CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS + ); + let r2 = RendererConfig { + camoufox_challenge_wait_ms: Some(4_321), + ..Default::default() + }; + assert_eq!(r2.camoufox_challenge_wait(), 4_321); + // Explicit 0 must disable the poll, not fall back to the default. + let r3 = RendererConfig { + camoufox_challenge_wait_ms: Some(0), + ..Default::default() + }; + assert_eq!(r3.camoufox_challenge_wait(), 0); + } + #[test] fn cloak_timeout_default_and_override() { let r = RendererConfig::default(); diff --git a/crates/crw-renderer/src/camoufox.rs b/crates/crw-renderer/src/camoufox.rs index eb6dc0bd..58b61525 100644 --- a/crates/crw-renderer/src/camoufox.rs +++ b/crates/crw-renderer/src/camoufox.rs @@ -25,6 +25,7 @@ use async_trait::async_trait; use crw_core::Deadline; +use crw_core::config::CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS; use crw_core::error::{CrwError, CrwResult}; use crw_core::types::FetchResult; use std::collections::HashMap; @@ -40,6 +41,11 @@ const OUTER_HTML_EXPR: &str = "document.documentElement.outerHTML"; const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5); /// Budget for the `is_available` health probe. const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +/// Delay between polls while a bot challenge is clearing. Same cadence as the +/// CDP tiers' `cdp::CHALLENGE_POLL_INTERVAL_MS`; kept as a separate constant +/// because the `camoufox` feature does not imply `cdp`, so that module is not +/// compiled in a camoufox-only build. +const CHALLENGE_POLL_INTERVAL: Duration = Duration::from_secs(3); /// Opt-in Camoufox stealth renderer. Construct via [`CamoufoxRenderer::new`]. pub struct CamoufoxRenderer { @@ -50,6 +56,9 @@ pub struct CamoufoxRenderer { api_key: String, /// Overall per-request REST budget (`config.camoufox_timeout()`). timeout: Duration, + /// Ceiling for polling a tab while a bot challenge clears + /// (`config.camoufox_challenge_wait()`). `ZERO` disables the poll. + challenge_wait: Duration, client: reqwest::Client, } @@ -60,10 +69,19 @@ impl CamoufoxRenderer { base_url: base_url.trim_end_matches('/').to_string(), api_key: api_key.to_string(), timeout: Duration::from_millis(timeout_ms), + challenge_wait: Duration::from_millis(CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS), client: reqwest::Client::new(), } } + /// Override the bot-challenge poll ceiling. Mirrors + /// `CdpRenderer::with_challenge_retries`, so the tier is configured the + /// same way the CDP tiers are. `0` disables the poll. + pub fn with_challenge_wait(mut self, challenge_wait_ms: u64) -> Self { + self.challenge_wait = Duration::from_millis(challenge_wait_ms); + self + } + /// Attach the bearer header when an API key is configured. fn auth(&self, rb: reqwest::RequestBuilder) -> reqwest::RequestBuilder { if self.api_key.is_empty() { @@ -228,7 +246,36 @@ impl CamoufoxRenderer { deadline: &Deadline, ) -> CrwResult<(u16, String)> { let tab_id = self.create_tab(url, user_id, session_key, deadline).await?; - let html = self.evaluate_outer_html(&tab_id, user_id, deadline).await?; + let mut html = self.evaluate_outer_html(&tab_id, user_id, deadline).await?; + + // A Cloudflare-style JS challenge resolves client-side, empirically + // 5-25s after navigation. A single immediate evaluate() can therefore + // only ever observe the interstitial, and this tier reported a wall for + // pages it would have gotten had it looked again. Poll until the + // challenge clears or the budget runs out. + // + // Only `"challenge"` is polled, never `"wall"`. The wall markers + // ("attention required! | cloudflare", "enable javascript and cookies + // to continue") are terminal refusals rather than work in progress — + // waiting on those would spend the whole budget to arrive at the same + // error. + // + // The ceiling is clamped by the deadline's own remaining time, so a + // challenge that never clears cannot outlive the request. A clean page + // never enters the loop: `looks_like_wall` is `None` on the first + // evaluate, and a deployment that sets the ceiling to 0 keeps exactly + // today's single-shot behaviour. + let challenge_budget = self.challenge_wait.min(deadline.remaining()); + let poll_start = Instant::now(); + while looks_like_wall(&html) == Some("challenge") { + let remaining = challenge_budget.saturating_sub(poll_start.elapsed()); + if remaining.is_zero() { + break; + } + tokio::time::sleep(CHALLENGE_POLL_INTERVAL.min(remaining)).await; + html = self.evaluate_outer_html(&tab_id, user_id, deadline).await?; + } + // A bot wall or an empty body is a failure for THIS tier — surface it as // a retryable RendererError so the fallback loop / breaker can react. if html.trim().is_empty() { @@ -564,7 +611,11 @@ mod tests { .await; mount_delete_session(&server).await; - let r = renderer(&server.uri()); + // Tiny challenge ceiling: this challenge never clears, so the poll must + // exhaust its budget. 50ms keeps that in the millisecond range instead + // of spending the production ceiling in CI, while still running the + // loop body at least once. + let r = renderer(&server.uri()).with_challenge_wait(50); let err = r .fetch("https://example.com", &HashMap::new(), None, deadline()) .await @@ -575,6 +626,93 @@ mod tests { } } + /// The point of the poll: a challenge that clears between evaluates now + /// yields the real page instead of the interstitial. + #[tokio::test] + async fn challenge_that_clears_on_a_later_poll_is_returned_as_content() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + // First evaluate: the interstitial. Registered first and capped at one + // call, so the second evaluate falls through to the real page below. + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Just a moment..." + }))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "

real page

" + }))) + .mount(&server) + .await; + mount_delete_session(&server).await; + + // 200ms ceiling: one poll, sub-second in CI. The loop sleeps + // `CHALLENGE_POLL_INTERVAL.min(remaining)`, so the budget bounds the + // wait rather than the production cadence. + let r = renderer(&server.uri()).with_challenge_wait(200); + let res = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect("challenge cleared, so the page must be returned"); + assert_eq!(res.status_code, 200); + assert!(res.html.contains("real page"), "got: {}", res.html); + } + + /// `looks_like_wall` separates a clearing "challenge" from a terminal + /// "wall". Only the former is worth polling; a wall must fail immediately + /// rather than burn the whole ceiling to reach the same error. + #[tokio::test] + async fn terminal_wall_is_not_polled() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Attention Required! | Cloudflare" + }))) + .mount(&server) + .await; + mount_delete_session(&server).await; + + // A ceiling far larger than one poll interval: if the wall were polled + // this would take seconds. + let r = renderer(&server.uri()).with_challenge_wait(30_000); + let started = Instant::now(); + let err = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect_err("a terminal wall is still an error"); + match err { + CrwError::RendererError(m) => assert!(m.contains("wall"), "got: {m}"), + other => panic!("expected RendererError, got {other:?}"), + } + assert!( + started.elapsed() < CHALLENGE_POLL_INTERVAL, + "terminal wall must not be polled, took {:?}", + started.elapsed() + ); + } + #[tokio::test] async fn expired_deadline_short_circuits_without_http() { // No mocks mounted: if any HTTP call were made it would 404 and the diff --git a/crates/crw-renderer/src/lib.rs b/crates/crw-renderer/src/lib.rs index 23769695..d1294d61 100644 --- a/crates/crw-renderer/src/lib.rs +++ b/crates/crw-renderer/src/lib.rs @@ -986,12 +986,15 @@ impl FallbackRenderer { .as_ref() .filter(|c| !c.base_url.trim().is_empty()) { - js_renderers.push(Arc::new(camoufox::CamoufoxRenderer::new( - "camoufox", - &cf.base_url, - &cf.api_key, - config.camoufox_timeout(), - )) as Arc); + js_renderers.push(Arc::new( + camoufox::CamoufoxRenderer::new( + "camoufox", + &cf.base_url, + &cf.api_key, + config.camoufox_timeout(), + ) + .with_challenge_wait(config.camoufox_challenge_wait()), + ) as Arc); tracing::info!( base_url = %cf.base_url, include_in_auto = cf.include_in_auto, diff --git a/docs/docs/js-rendering.md b/docs/docs/js-rendering.md index 73895ab0..020ed42a 100644 --- a/docs/docs/js-rendering.md +++ b/docs/docs/js-rendering.md @@ -235,6 +235,10 @@ configured endpoint stays out of the `auto` failover chain until you ask for it. # # if you front the sidecar with an auth proxy # include_in_auto = false # default: stay OUT of the auto ladder # camoufox_timeout_ms = 60000 # per-request REST budget (default 60s) + # camoufox_challenge_wait_ms = 20000 + # # ceiling for polling a tab while a Cloudflare-style + # # JS challenge clears itself (default 20s; 0 disables + # # the poll and reports the interstitial immediately) ``` ### Three ways to use it From 04af5888bb58b2ca917f7bd1689bfbbb72601fe9 Mon Sep 17 00:00:00 2001 From: us <22618852+us@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:21:19 +0200 Subject: [PATCH 2/3] fix(renderer): stop camoufox polling out a page that already cleared The predicate the new loop polls on matched the bare `/cdn-cgi/challenge-platform` directory. Cloudflare re-injects that telemetry loader (`scripts/jsd/main.js`) into pages that have ALREADY cleared, so on a managed site camoufox renders the real page, the predicate still reads "challenge", and the loop spends the whole ceiling before discarding it. That marker was removed from crw_crawl::single::classify_block and from detector::looks_like_cloudflare_challenge for exactly this reason, each after a live capture; this list had kept it. Narrowed to `challenge-platform/h/`, the orchestrator path, which the telemetry loader never uses. Terminal walls are now matched first. `find` returns the first needle in LIST order, and every challenge needle preceded both wall needles, so a page carrying both classified as a clearing challenge and got polled for the whole ceiling to arrive at the same refusal, which is what the loop's comment says it avoids. The deadline is re-checked after the sleep. It could otherwise be spent by the sleep itself, and the next evaluate would then return `Timeout`, replacing this tier's wall error and its antibot attribution with a bare "timed out". Tests: the cleared-page fixture now carries the telemetry loader, so it fails against the old marker; a page carrying both marker kinds pins wall precedence; and the terminal-wall test counts evaluate calls instead of asserting on wall-clock, which cannot flake on a loaded runner. --- crates/crw-renderer/src/camoufox.rs | 95 +++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 20 deletions(-) diff --git a/crates/crw-renderer/src/camoufox.rs b/crates/crw-renderer/src/camoufox.rs index 58b61525..abf4081a 100644 --- a/crates/crw-renderer/src/camoufox.rs +++ b/crates/crw-renderer/src/camoufox.rs @@ -273,6 +273,14 @@ impl CamoufoxRenderer { break; } tokio::time::sleep(CHALLENGE_POLL_INTERVAL.min(remaining)).await; + // The sleep can consume what was left of the shared deadline. Stop + // here rather than dispatching an evaluate with no budget: that call + // would return `Timeout`, replacing this tier's informative wall + // error (and the antibot attribution that rides on it) with a bare + // "timed out". + if deadline.remaining().is_zero() { + break; + } html = self.evaluate_outer_html(&tab_id, user_id, deadline).await?; } @@ -300,19 +308,36 @@ impl CamoufoxRenderer { /// report a retryable failure instead of returning a useless challenge page. fn looks_like_wall(html: &str) -> Option<&'static str> { let h = html.to_ascii_lowercase(); - const NEEDLES: &[(&str, &str)] = &[ - ("just a moment", "challenge"), - ("verifying you are human", "challenge"), - ("checking your browser before", "challenge"), - ("cf-challenge", "challenge"), - ("/cdn-cgi/challenge-platform", "challenge"), - ("attention required! | cloudflare", "wall"), - ("enable javascript and cookies to continue", "wall"), + // Terminal refusals are checked FIRST. A page routinely carries both kinds + // of marker, and matching in list order would classify such a page as a + // clearing challenge and poll it for the whole ceiling to arrive at the same + // refusal. A wall is not work in progress, so it wins. + const WALLS: &[&str] = &[ + "attention required! | cloudflare", + "enable javascript and cookies to continue", ]; - NEEDLES - .iter() - .find(|(needle, _)| h.contains(needle)) - .map(|(_, kind)| *kind) + if WALLS.iter().any(|needle| h.contains(needle)) { + return Some("wall"); + } + // `challenge-platform/h/`, not the bare `/cdn-cgi/challenge-platform` + // directory. Both live under it and they mean opposite things: the + // orchestrator is always served from `/h/`, while `scripts/jsd/main.js` is + // the ordinary Bot-Management loader that Cloudflare re-injects into pages + // that have ALREADY cleared. Matching the bare directory therefore fires on + // a solved page. `crw_crawl::single::classify_block` and + // `detector::looks_like_cloudflare_challenge` both dropped it for exactly + // that reason, each after a live capture; this list had kept it. + const CHALLENGES: &[&str] = &[ + "just a moment", + "verifying you are human", + "checking your browser before", + "cf-challenge", + "challenge-platform/h/", + ]; + if CHALLENGES.iter().any(|needle| h.contains(needle)) { + return Some("challenge"); + } + None } #[async_trait] @@ -649,11 +674,16 @@ mod tests { .up_to_n_times(1) .mount(&server) .await; + // The cleared page, shaped like a real one: Cloudflare re-injects the + // Bot-Management telemetry loader into pages that have already passed, + // so a fixture without it would let a predicate that matches the bare + // `/cdn-cgi/challenge-platform` directory pass this test while polling + // every real managed site to the ceiling and then discarding it. Mock::given(method("POST")) .and(path("/tabs/t1/evaluate")) .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ "ok": true, - "result": "

real page

" + "result": "

real page

" }))) .mount(&server) .await; @@ -697,7 +727,6 @@ mod tests { // A ceiling far larger than one poll interval: if the wall were polled // this would take seconds. let r = renderer(&server.uri()).with_challenge_wait(30_000); - let started = Instant::now(); let err = r .fetch("https://example.com", &HashMap::new(), None, deadline()) .await @@ -706,11 +735,17 @@ mod tests { CrwError::RendererError(m) => assert!(m.contains("wall"), "got: {m}"), other => panic!("expected RendererError, got {other:?}"), } - assert!( - started.elapsed() < CHALLENGE_POLL_INTERVAL, - "terminal wall must not be polled, took {:?}", - started.elapsed() - ); + // Count the evaluates rather than the elapsed time: a wall-clock assert + // on a loaded runner is a flake, and the request count proves the same + // thing exactly. One evaluate means the loop was never entered. + let evaluates = server + .received_requests() + .await + .expect("recorded requests") + .iter() + .filter(|r| r.url.path() == "/tabs/t1/evaluate") + .count(); + assert_eq!(evaluates, 1, "a terminal wall must not be polled"); } #[tokio::test] @@ -759,9 +794,29 @@ mod tests { fn wall_needles_match_and_clean_html_passes() { assert_eq!(looks_like_wall("Just a moment..."), Some("challenge")); assert_eq!( - looks_like_wall("\ +
Real content served after the challenge cleared.
\ + " + ), + None + ); + // A page carrying both kinds of marker is a refusal, not work in + // progress, so it must classify as a wall and fail immediately. + assert_eq!( + looks_like_wall( + "

Verifying you are human

\ +

Enable JavaScript and cookies to continue

" + ), + Some("wall") + ); assert_eq!( looks_like_wall("
"), Some("challenge") From 2709116f54274bf1b1ac913fab1bebccddd05c56 Mon Sep 17 00:00:00 2001 From: us Date: Tue, 8 Sep 2026 13:48:22 +0300 Subject: [PATCH 3/3] fix(renderer): poll the interstitial camoufox actually sees The wall-first ordering classified the standard Cloudflare managed-challenge interstitial as a terminal wall: "enable javascript and cookies to continue" is the noscript line that page ships, and the repo's own capture in tests/egress_latch_no_latch_on_cf.rs carries it next to the orchestrator script. The poll loop therefore never ran on the page class it was written for. Only the "Attention Required" title is terminal-first now; the noscript line is a refusal only when no challenge marker is present and it sits outside