diff --git a/config.default.toml b/config.default.toml index b659c438..85f78e72 100644 --- a/config.default.toml +++ b/config.default.toml @@ -28,6 +28,10 @@ rate_limit_rps = 10 # Max requests/second (global). 0 = unlimited. mode = "auto" # auto | lightpanda | playwright | chrome | camoufox | none page_timeout_ms = 30000 pool_size = 4 +# camoufox_timeout_ms = 60000 # camoufox per-request REST budget override +# camoufox_challenge_wait_ms = 20000 +# # ceiling for polling out a Cloudflare-style JS +# # challenge on the camoufox tier (0 disables) # render_js_default = true # alias: force_js = true # Forces JS rendering when a request omits `renderJs`. # Per-request `renderJs` always wins over this default. @@ -65,7 +69,10 @@ ws_url = "ws://127.0.0.1:9222/" # base_url = "http://127.0.0.1:9377" # api_key = "" # optional bearer token if the sidecar is protected # include_in_auto = false # default: stay out of the auto ladder -# camoufox_timeout_ms = 60000 # optional per-request REST budget override +# +# The two camoufox budgets (camoufox_timeout_ms, camoufox_challenge_wait_ms) +# are top-level [renderer] keys, shown in that section above. Placed under +# [renderer.camoufox] they are silently ignored. [crawler] max_concurrency = 10 diff --git a/crates/crw-core/src/config.rs b/crates/crw-core/src/config.rs index 63fc6670..db6003eb 100644 --- a/crates/crw-core/src/config.rs +++ b/crates/crw-core/src/config.rs @@ -221,6 +221,15 @@ pub const MAX_WAIT_FOR_MS: u64 = 60_000; /// and must never be charged CDP overhead. pub const CAMOUFOX_DEFAULT_TIMEOUT_MS: u64 = 60_000; +/// Default ceiling (ms) for polling a Camoufox tab while a Cloudflare-style JS +/// challenge clears itself, before giving up and reporting the wall. The CDP +/// tiers already run this loop ([`crw-renderer`'s `CHALLENGE_MAX_RETRIES`] × +/// `CHALLENGE_POLL_INTERVAL_MS` = 9s); Camoufox is a slower engine reached +/// after the CDP tiers have already failed, so it gets a wider ceiling. 20s +/// covers the commonly observed 5-25s clear without eating most of a 60s +/// request budget. Used by [`RendererConfig::camoufox_challenge_wait`]. +pub const CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS: u64 = 20_000; + /// Default cloak (Turnstile-solver sidecar) per-request budget (ms) — the /// per-attempt solve budget bounding one sidecar mirror call. A cold interactive /// Turnstile solve is ~21s; 35s leaves margin for the browser solve + curl_cffi @@ -803,6 +812,14 @@ pub struct RendererConfig { /// [`CAMOUFOX_DEFAULT_TIMEOUT_MS`] when unset. #[serde(default)] pub camoufox_timeout_ms: Option, + /// Ceiling (ms) for polling a Camoufox tab while a bot-challenge + /// interstitial clears, before giving up and reporting the wall. Falls back + /// to [`CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS`] when unset; `Some(0)` disables + /// the poll and restores the single-shot behaviour. The Camoufox analogue + /// of `chrome_challenge_max_retries`. + /// Env: `CRW_RENDERER__CAMOUFOX_CHALLENGE_WAIT_MS`. + #[serde(default)] + pub camoufox_challenge_wait_ms: Option, /// Opt-in cloak Turnstile-solver sidecar endpoint. See [`CloakEndpoint`]. /// `None` = not configured (default) → the tier is never constructed and the /// engine is byte-identical to a build without it. Fired only as a @@ -1080,6 +1097,7 @@ impl Default for RendererConfig { chrome_proxy_timeout_ms: None, camoufox: None, camoufox_timeout_ms: None, + camoufox_challenge_wait_ms: None, cloak: None, cloak_timeout_ms: None, cloak_proxy_host: None, @@ -1133,6 +1151,12 @@ impl RendererConfig { self.camoufox_timeout_ms .unwrap_or(CAMOUFOX_DEFAULT_TIMEOUT_MS) } + /// Camoufox bot-challenge poll ceiling (ms). Unconditional (no `#[cfg]`), + /// like [`Self::camoufox_timeout`]. + pub fn camoufox_challenge_wait(&self) -> u64 { + self.camoufox_challenge_wait_ms + .unwrap_or(CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS) + } /// Per-attempt cloak solve budget (ms). Unconditional (no `#[cfg]`) so /// `tier_timeouts_from` can reference it in every build, like /// [`Self::camoufox_timeout`]. @@ -3171,6 +3195,7 @@ search_backend_url = "http://from-file:8080" assert_eq!(r.chrome_proxy_timeout_ms, None); assert!(r.camoufox.is_none()); assert_eq!(r.camoufox_timeout_ms, None); + assert_eq!(r.camoufox_challenge_wait_ms, None); assert!(r.cloak.is_none()); assert_eq!(r.cloak_timeout_ms, None); assert_eq!(r.cloak_proxy_host, None); @@ -3355,6 +3380,26 @@ search_backend_url = "http://from-file:8080" assert_eq!(r2.camoufox_timeout(), 1_234); } + #[test] + fn camoufox_challenge_wait_default_and_override() { + let r = RendererConfig::default(); + assert_eq!( + r.camoufox_challenge_wait(), + CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS + ); + let r2 = RendererConfig { + camoufox_challenge_wait_ms: Some(4_321), + ..Default::default() + }; + assert_eq!(r2.camoufox_challenge_wait(), 4_321); + // Explicit 0 must disable the poll, not fall back to the default. + let r3 = RendererConfig { + camoufox_challenge_wait_ms: Some(0), + ..Default::default() + }; + assert_eq!(r3.camoufox_challenge_wait(), 0); + } + #[test] fn cloak_timeout_default_and_override() { let r = RendererConfig::default(); diff --git a/crates/crw-renderer/src/camoufox.rs b/crates/crw-renderer/src/camoufox.rs index eb6dc0bd..659c67b2 100644 --- a/crates/crw-renderer/src/camoufox.rs +++ b/crates/crw-renderer/src/camoufox.rs @@ -25,6 +25,7 @@ use async_trait::async_trait; use crw_core::Deadline; +use crw_core::config::CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS; use crw_core::error::{CrwError, CrwResult}; use crw_core::types::FetchResult; use std::collections::HashMap; @@ -40,6 +41,22 @@ const OUTER_HTML_EXPR: &str = "document.documentElement.outerHTML"; const CLEANUP_TIMEOUT: Duration = Duration::from_secs(5); /// Budget for the `is_available` health probe. const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +/// Delay between polls while a bot challenge is clearing. Same cadence as the +/// CDP tiers' `cdp::CHALLENGE_POLL_INTERVAL_MS`; kept as a separate constant +/// because the `camoufox` feature does not imply `cdp`, so that module is not +/// compiled in a camoufox-only build. +const CHALLENGE_POLL_INTERVAL: Duration = Duration::from_secs(3); +/// Smallest request-deadline remainder worth spending on one more poll +/// evaluate. Below it the call can only time out, which would replace the +/// informative wall error with a bare "timed out". +const MIN_POLL_EVALUATE_BUDGET: Duration = Duration::from_millis(250); +/// What the poll can still spend after its ceiling before the caller sees the +/// result: one last evaluate at its floor plus the awaited session teardown. +/// A reserve handed to [`CamoufoxRenderer::with_challenge_reserve`] on behalf +/// of something that runs after this tier has to include it. +#[cfg(feature = "cloak")] +pub(crate) const POLL_HANDOFF_MS: u64 = + MIN_POLL_EVALUATE_BUDGET.as_millis() as u64 + CLEANUP_TIMEOUT.as_millis() as u64; /// Opt-in Camoufox stealth renderer. Construct via [`CamoufoxRenderer::new`]. pub struct CamoufoxRenderer { @@ -50,6 +67,14 @@ pub struct CamoufoxRenderer { api_key: String, /// Overall per-request REST budget (`config.camoufox_timeout()`). timeout: Duration, + /// Ceiling for polling a tab while a bot challenge clears + /// (`config.camoufox_challenge_wait()`). `ZERO` disables the poll. + challenge_wait: Duration, + /// Request-deadline time the poll must leave untouched, so a recovery arm + /// that runs after the ladder (the cloak arm, gated on + /// `CLOAK_ARM_FLOOR_MS`) still finds its floor. Zero when nothing runs + /// after this tier. + challenge_reserve: Duration, client: reqwest::Client, } @@ -60,10 +85,28 @@ impl CamoufoxRenderer { base_url: base_url.trim_end_matches('/').to_string(), api_key: api_key.to_string(), timeout: Duration::from_millis(timeout_ms), + challenge_wait: Duration::from_millis(CAMOUFOX_DEFAULT_CHALLENGE_WAIT_MS), + challenge_reserve: Duration::ZERO, client: reqwest::Client::new(), } } + /// Override the bot-challenge poll ceiling. Mirrors + /// `CdpRenderer::with_challenge_retries`, so the tier is configured the + /// same way the CDP tiers are. `0` disables the poll. + pub fn with_challenge_wait(mut self, challenge_wait_ms: u64) -> Self { + self.challenge_wait = Duration::from_millis(challenge_wait_ms); + self + } + + /// Keep `reserve_ms` of the request deadline out of the challenge poll. + /// The ladder sets this when a recovery arm runs after camoufox, so the + /// poll cannot spend the floor that arm needs to fire. + pub fn with_challenge_reserve(mut self, reserve_ms: u64) -> Self { + self.challenge_reserve = Duration::from_millis(reserve_ms); + self + } + /// Attach the bearer header when an API key is configured. fn auth(&self, rb: reqwest::RequestBuilder) -> reqwest::RequestBuilder { if self.api_key.is_empty() { @@ -228,7 +271,90 @@ impl CamoufoxRenderer { deadline: &Deadline, ) -> CrwResult<(u16, String)> { let tab_id = self.create_tab(url, user_id, session_key, deadline).await?; - let html = self.evaluate_outer_html(&tab_id, user_id, deadline).await?; + let mut html = self.evaluate_outer_html(&tab_id, user_id, deadline).await?; + + // A Cloudflare-style JS challenge resolves client-side, empirically + // 5-25s after navigation. A single immediate evaluate() can therefore + // only ever observe the interstitial, and this tier reported a wall for + // pages it would have gotten had it looked again. Poll until the + // challenge clears or the budget runs out. + // + // Only `"challenge"` is polled, never `"wall"`. A wall is a terminal + // refusal rather than work in progress: waiting on it would spend the + // whole budget to arrive at the same error. + // + // The ceiling is clamped by the deadline's own remaining time, minus + // whatever a recovery arm after this tier needs, so a challenge that + // never clears cannot outlive the request or starve that arm. A clean + // page never enters the loop: `looks_like_wall` is `None` on the first + // evaluate, and a deployment that sets the ceiling to 0 keeps exactly + // today's single-shot behaviour. + // The reserve only applies while there is more deadline left than the + // reserve itself. Below that the poll is deliberately preferred over + // the arm it would protect: holding the whole remainder back would + // leave this tier a single look, and the arm may not be reachable at + // all on this request. + let remaining_now = deadline.remaining(); + let remaining_for_poll = if remaining_now > self.challenge_reserve { + remaining_now - self.challenge_reserve + } else { + remaining_now + }; + let challenge_budget = self.challenge_wait.min(remaining_for_poll); + let poll_start = Instant::now(); + let mut missed_poll = false; + while looks_like_wall(&html) == Some("challenge") { + let remaining = challenge_budget.saturating_sub(poll_start.elapsed()); + if remaining.is_zero() { + break; + } + tokio::time::sleep(CHALLENGE_POLL_INTERVAL.min(remaining)).await; + // The sleep can consume what was left of the shared deadline. Stop + // here rather than dispatching an evaluate that cannot finish: that + // call would return `Timeout`, replacing this tier's informative + // wall error (and the antibot attribution that rides on it) with a + // bare "timed out". + if deadline.remaining() < MIN_POLL_EVALUATE_BUDGET { + break; + } + // The challenge clears by reloading the tab, and an evaluate that + // lands in that window fails ("execution context destroyed", + // surfaced as ok:false or a non-2xx). One such miss is part of the + // poll, not a failure of the tier; two in a row means the sidecar + // itself is broken and the error is the honest answer. The flag + // resets on every good evaluate, so it tolerates one miss per + // streak, bounded by the ceiling. A `Timeout` counts as a miss too: + // the loop then ends on the budget and reports the challenge it + // saw rather than a bare "timed out". + // Bound the evaluate by what is left of the ceiling (with a small + // floor so the final look can still complete), not by the whole + // request deadline: otherwise one slow evaluate could spend the + // reserve the ceiling was clamped to keep. + let poll_left = challenge_budget.saturating_sub(poll_start.elapsed()); + let evaluate_deadline = Deadline::now_plus( + poll_left + .max(MIN_POLL_EVALUATE_BUDGET) + .min(deadline.remaining()), + ); + match self + .evaluate_outer_html(&tab_id, user_id, &evaluate_deadline) + .await + { + Ok(next) => { + html = next; + missed_poll = false; + } + Err(e) if !missed_poll => { + tracing::debug!( + renderer = %self.name, + "camoufox: challenge poll evaluate failed, retrying once: {e}" + ); + missed_poll = true; + } + Err(e) => return Err(e), + } + } + // A bot wall or an empty body is a failure for THIS tier — surface it as // a retryable RendererError so the fallback loop / breaker can react. if html.trim().is_empty() { @@ -253,19 +379,64 @@ impl CamoufoxRenderer { /// report a retryable failure instead of returning a useless challenge page. fn looks_like_wall(html: &str) -> Option<&'static str> { let h = html.to_ascii_lowercase(); - const NEEDLES: &[(&str, &str)] = &[ - ("just a moment", "challenge"), - ("verifying you are human", "challenge"), - ("checking your browser before", "challenge"), - ("cf-challenge", "challenge"), - ("/cdn-cgi/challenge-platform", "challenge"), - ("attention required! | cloudflare", "wall"), - ("enable javascript and cookies to continue", "wall"), + // The one terminal refusal is checked FIRST: the "Attention Required" + // title fronts the 1020 access-denied, the 1009 country ban and the + // interactive captcha, none of which clears by waiting, and it can share a + // page with a challenge marker (the legacy interstitial paired it with + // "checking your browser"). Matching in list order would poll such a page + // for the whole ceiling to arrive at the same refusal. The CDP tier's + // `is_challenge_page` does poll this title; the divergence is deliberate, + // since this tier is normally the last one and its poll is far longer. + if h.contains("attention required! | cloudflare") { + return Some("wall"); + } + // `challenge-platform/h/`, not the bare `/cdn-cgi/challenge-platform` + // directory. Both live under it and they mean opposite things: the + // orchestrator is always served from `/h/`, while `scripts/jsd/main.js` is + // the ordinary Bot-Management loader that Cloudflare re-injects into pages + // that have ALREADY cleared. Matching the bare directory therefore fires on + // a solved page. `crw_crawl::single::classify_block` and + // `detector::looks_like_cloudflare_challenge` both dropped it for exactly + // that reason, each after a live capture; this list had kept it. + const CHALLENGES: &[&str] = &[ + "just a moment", + "verifying you are human", + "checking your browser before", + "cf-challenge", + "challenge-platform/h/", ]; - NEEDLES - .iter() - .find(|(needle, _)| h.contains(needle)) - .map(|(_, kind)| *kind) + if CHALLENGES.iter().any(|needle| h.contains(needle)) { + return Some("challenge"); + } + // The noscript line ships ON the managed-challenge interstitial itself + // (`crates/crw-renderer/tests/egress_latch_no_latch_on_cf.rs` carries the + // canonical capture), so it must never outrank a challenge marker. Alone, + // with no challenge around it, it is a plain refusal, unless it only sits + // inside `".len()..], + None => "", + }; + } + out.push_str(rest); + std::borrow::Cow::Owned(out) } #[async_trait] @@ -564,7 +735,11 @@ mod tests { .await; mount_delete_session(&server).await; - let r = renderer(&server.uri()); + // Tiny challenge ceiling: this challenge never clears, so the poll must + // exhaust its budget. 50ms keeps that in the millisecond range instead + // of spending the production ceiling in CI, while still running the + // loop body at least once. + let r = renderer(&server.uri()).with_challenge_wait(50); let err = r .fetch("https://example.com", &HashMap::new(), None, deadline()) .await @@ -575,6 +750,254 @@ mod tests { } } + /// The point of the poll: a challenge that clears between evaluates now + /// yields the real page instead of the interstitial. + #[tokio::test] + async fn challenge_that_clears_on_a_later_poll_is_returned_as_content() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + // First evaluate: the interstitial. Registered first and capped at one + // call, so the second evaluate falls through to the real page below. + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Just a moment..." + }))) + .up_to_n_times(1) + .mount(&server) + .await; + // The cleared page, shaped like a real one: Cloudflare re-injects the + // Bot-Management telemetry loader into pages that have already passed, + // so a fixture without it would let a predicate that matches the bare + // `/cdn-cgi/challenge-platform` directory pass this test while polling + // every real managed site to the ceiling and then discarding it. + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "

real page

" + }))) + .mount(&server) + .await; + mount_delete_session(&server).await; + + // 200ms ceiling: one poll, sub-second in CI. The loop sleeps + // `CHALLENGE_POLL_INTERVAL.min(remaining)`, so the budget bounds the + // wait rather than the production cadence. + let r = renderer(&server.uri()).with_challenge_wait(200); + let res = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect("challenge cleared, so the page must be returned"); + assert_eq!(res.status_code, 200); + assert!(res.html.contains("real page"), "got: {}", res.html); + } + + /// A challenge clears by reloading the tab, so an evaluate that lands in + /// that window fails. One such miss must not end the tier: the next poll + /// picks up the cleared page. + #[tokio::test] + async fn evaluate_failure_mid_poll_is_a_miss_not_a_failure() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Just a moment..." + }))) + .up_to_n_times(1) + .mount(&server) + .await; + // Second evaluate: the context was destroyed by the reload. + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"ok": false})), + ) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "

real page

" + }))) + .mount(&server) + .await; + mount_delete_session(&server).await; + + // One full poll interval for the failed look, then the remainder for + // the look that finds the cleared page, with slack for a loaded runner. + let r = renderer(&server.uri()).with_challenge_wait(4_500); + let res = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect("one failed poll must not fail the tier"); + assert!(res.html.contains("real page"), "got: {}", res.html); + } + + /// Two failed polls in a row mean the sidecar is broken, and that error is + /// the honest answer rather than a wall the page never showed. + #[tokio::test] + async fn two_consecutive_evaluate_failures_surface_the_error() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Just a moment..." + }))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"ok": false})), + ) + .mount(&server) + .await; + mount_delete_session(&server).await; + + let r = renderer(&server.uri()).with_challenge_wait(30_000); + let err = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect_err("a broken sidecar is an error"); + match err { + CrwError::RendererError(m) => assert!(m.contains("ok=false"), "got: {m}"), + other => panic!("expected RendererError, got {other:?}"), + } + let evaluates = server + .received_requests() + .await + .expect("recorded requests") + .iter() + .filter(|r| r.url.path() == "/tabs/t1/evaluate") + .count(); + assert_eq!( + evaluates, 3, + "interstitial, one tolerated miss, then the error" + ); + } + + /// The reserve keeps the poll from spending the deadline a recovery arm + /// after this tier needs: with almost all of it reserved, the poll stops + /// after a single look instead of running out the 30s ceiling. + #[tokio::test] + async fn challenge_reserve_bounds_the_poll() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Just a moment..." + }))) + .mount(&server) + .await; + mount_delete_session(&server).await; + + // 30s deadline, 30s ceiling, 29.9s reserved: 100ms of poll budget. + let r = renderer(&server.uri()) + .with_challenge_wait(30_000) + .with_challenge_reserve(29_900); + let err = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect_err("the challenge never clears"); + match err { + CrwError::RendererError(m) => assert!(m.contains("challenge"), "got: {m}"), + other => panic!("expected RendererError, got {other:?}"), + } + let evaluates = server + .received_requests() + .await + .expect("recorded requests") + .iter() + .filter(|r| r.url.path() == "/tabs/t1/evaluate") + .count(); + assert!( + evaluates <= 2, + "a 100ms budget allows at most one poll, got {evaluates}" + ); + } + + /// `looks_like_wall` separates a clearing "challenge" from a terminal + /// "wall". Only the former is worth polling; a wall must fail immediately + /// rather than burn the whole ceiling to reach the same error. + #[tokio::test] + async fn terminal_wall_is_not_polled() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/tabs")) + .respond_with( + ResponseTemplate::new(200).set_body_json(serde_json::json!({"tabId": "t1"})), + ) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/tabs/t1/evaluate")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "ok": true, + "result": "Attention Required! | Cloudflare" + }))) + .mount(&server) + .await; + mount_delete_session(&server).await; + + // A ceiling far larger than one poll interval: if the wall were polled + // this would take seconds. + let r = renderer(&server.uri()).with_challenge_wait(30_000); + let err = r + .fetch("https://example.com", &HashMap::new(), None, deadline()) + .await + .expect_err("a terminal wall is still an error"); + match err { + CrwError::RendererError(m) => assert!(m.contains("wall"), "got: {m}"), + other => panic!("expected RendererError, got {other:?}"), + } + // Count the evaluates rather than the elapsed time: a wall-clock assert + // on a loaded runner is a flake, and the request count proves the same + // thing exactly. One evaluate means the loop was never entered. + let evaluates = server + .received_requests() + .await + .expect("recorded requests") + .iter() + .filter(|r| r.url.path() == "/tabs/t1/evaluate") + .count(); + assert_eq!(evaluates, 1, "a terminal wall must not be polled"); + } + #[tokio::test] async fn expired_deadline_short_circuits_without_http() { // No mocks mounted: if any HTTP call were made it would 404 and the @@ -621,9 +1044,60 @@ mod tests { fn wall_needles_match_and_clean_html_passes() { assert_eq!(looks_like_wall("Just a moment..."), Some("challenge")); assert_eq!( - looks_like_wall("\ +
Real content served after the challenge cleared.
\ + " + ), + None + ); + // The noscript line is part of the managed-challenge interstitial, so + // a page carrying it next to a challenge marker is work in progress, + // not a refusal. This is the repo's canonical Cloudflare capture, kept + // verbatim from tests/egress_latch_no_latch_on_cf.rs. + assert_eq!( + looks_like_wall(concat!( + "

Just a moment...

", + "", + "
Enable JavaScript and cookies to continue
" + )), + Some("challenge") + ); + assert_eq!( + looks_like_wall( + "

Verifying you are human

\ +

Enable JavaScript and cookies to continue

" + ), + Some("challenge") + ); + // Alone it is a plain refusal. + assert_eq!( + looks_like_wall("

Please enable JavaScript and cookies to continue

"), + Some("wall") + ); + // Only inside noscript on an otherwise rendered page: not a refusal. + assert_eq!( + looks_like_wall( + "\ +
Real content that rendered fine.
" + ), + None + ); + // The terminal title outranks a challenge marker on the same page. + assert_eq!( + looks_like_wall( + "Attention Required! | Cloudflare\ + " + ), + Some("wall") + ); assert_eq!( looks_like_wall("
"), Some("challenge") diff --git a/crates/crw-renderer/src/lib.rs b/crates/crw-renderer/src/lib.rs index 23769695..f854753a 100644 --- a/crates/crw-renderer/src/lib.rs +++ b/crates/crw-renderer/src/lib.rs @@ -697,6 +697,41 @@ impl std::fmt::Debug for FallbackRenderer { } } +/// How much of the request deadline the camoufox challenge poll must leave for +/// the cloak recovery arm, or `None` when that arm cannot fire after it. +/// +/// The arm exists whenever a cloak endpoint is configured (it is a recovery +/// arm, not a ladder tier, so its own `include_in_auto` does not decide this), +/// and it is gated on `CLOAK_ARM_FLOOR_MS` of remaining deadline only while +/// `cloak_recover_on_cf` is off. With that flag on it fires on its own fresh +/// budget, so reserving for it would only shorten the poll for nothing. +/// +/// It also only fires when an earlier tier left a Cloudflare challenge body +/// behind, and camoufox reports a challenge as an error, never as a body. So +/// the reserve pays off in exactly one shape: `mode = "auto"` with camoufox in +/// the ladder, where chrome ran first. A camoufox pin or `mode = "camoufox"` +/// runs nothing before it, the arm is unreachable, and holding time back would +/// only cut the poll short. A pin issued on a deployment that also has +/// camoufox in auto still carries the reserve; that is a per-instance ceiling +/// worth knowing, not worth plumbing. +/// +/// The floor is what the arm checks AFTER this tier has returned, so the +/// reserve also covers the last poll evaluate and the awaited session +/// teardown that sit between the end of the poll and that check. The tier +/// honours the reserve only while more than the reserve remains; with less, +/// it prefers its own poll to an arm that may not fire. +#[cfg(all(feature = "camoufox", feature = "cloak"))] +fn camoufox_challenge_reserve_ms(config: &RendererConfig) -> Option { + let cloak_configured = config + .cloak + .as_ref() + .is_some_and(|c| !c.base_url.trim().is_empty()); + let camoufox_after_a_ladder = + matches!(config.mode, RendererMode::Auto) && config.camoufox_in_ladder(); + (camoufox_after_a_ladder && cloak_configured && !config.cloak_recover_on_cf) + .then_some(crw_core::config::CLOAK_ARM_FLOOR_MS + camoufox::POLL_HANDOFF_MS) +} + impl FallbackRenderer { pub fn new( config: &RendererConfig, @@ -986,12 +1021,19 @@ impl FallbackRenderer { .as_ref() .filter(|c| !c.base_url.trim().is_empty()) { - js_renderers.push(Arc::new(camoufox::CamoufoxRenderer::new( + let camoufox_tier = camoufox::CamoufoxRenderer::new( "camoufox", &cf.base_url, &cf.api_key, config.camoufox_timeout(), - )) as Arc); + ) + .with_challenge_wait(config.camoufox_challenge_wait()); + #[cfg(feature = "cloak")] + let camoufox_tier = match camoufox_challenge_reserve_ms(config) { + Some(reserve) => camoufox_tier.with_challenge_reserve(reserve), + None => camoufox_tier, + }; + js_renderers.push(Arc::new(camoufox_tier) as Arc); tracing::info!( base_url = %cf.base_url, include_in_auto = cf.include_in_auto, @@ -7596,3 +7638,58 @@ mod tests { assert_eq!(health.get("http"), Some(&true)); } } + +#[cfg(all(test, feature = "camoufox", feature = "cloak"))] +mod camoufox_reserve_tests { + use super::camoufox_challenge_reserve_ms; + use crw_core::config::{ + CLOAK_ARM_FLOOR_MS, CamoufoxEndpoint, CloakEndpoint, RendererConfig, RendererMode, + }; + + fn cfg( + mode: RendererMode, + camoufox_in_auto: bool, + cloak_base_url: &str, + recover_on_cf: bool, + ) -> RendererConfig { + RendererConfig { + mode, + camoufox: Some(CamoufoxEndpoint { + base_url: "http://camoufox:9377".to_string(), + include_in_auto: camoufox_in_auto, + ..Default::default() + }), + cloak: Some(CloakEndpoint { + base_url: cloak_base_url.to_string(), + ..Default::default() + }), + cloak_recover_on_cf: recover_on_cf, + ..Default::default() + } + } + + /// The reserve exists only where the arm can fire after camoufox: auto + /// mode with camoufox in the ladder, a cloak endpoint held as the floor + /// gated recovery arm. A camoufox held out of auto, a camoufox mode, an + /// arm that fires on its own budget, or no cloak at all: no reserve. + #[test] + fn reserve_tracks_the_cloak_arm_floor_gate() { + let auto_in_ladder = cfg(RendererMode::Auto, true, "http://cloak:8000", false); + assert_eq!( + camoufox_challenge_reserve_ms(&auto_in_ladder), + Some(CLOAK_ARM_FLOOR_MS + crate::camoufox::POLL_HANDOFF_MS) + ); + let recover_on_cf = cfg(RendererMode::Auto, true, "http://cloak:8000", true); + assert_eq!(camoufox_challenge_reserve_ms(&recover_on_cf), None); + let held_out_of_auto = cfg(RendererMode::Auto, false, "http://cloak:8000", false); + assert_eq!(camoufox_challenge_reserve_ms(&held_out_of_auto), None); + let camoufox_mode = cfg(RendererMode::Camoufox, true, "http://cloak:8000", false); + assert_eq!(camoufox_challenge_reserve_ms(&camoufox_mode), None); + let no_cloak = cfg(RendererMode::Auto, true, " ", false); + assert_eq!(camoufox_challenge_reserve_ms(&no_cloak), None); + assert_eq!( + camoufox_challenge_reserve_ms(&RendererConfig::default()), + None + ); + } +} diff --git a/docs/docs/js-rendering.md b/docs/docs/js-rendering.md index 73895ab0..a77013a9 100644 --- a/docs/docs/js-rendering.md +++ b/docs/docs/js-rendering.md @@ -234,7 +234,21 @@ configured endpoint stays out of the `auto` failover chain until you ask for it. # api_key = "..." # sent as `Authorization: Bearer` — only needed # # if you front the sidecar with an auth proxy # include_in_auto = false # default: stay OUT of the auto ladder + ``` + + The two camoufox budgets go in your existing `[renderer]` table (the one + that holds `mode`), not in `[renderer.camoufox]` and not in a second + `[renderer]` header. Under `[renderer.camoufox]` they are silently ignored + and the defaults stay in force. + + ```toml + [renderer] + mode = "auto" # camoufox_timeout_ms = 60000 # per-request REST budget (default 60s) + # camoufox_challenge_wait_ms = 20000 + # # ceiling for polling a tab while a Cloudflare-style + # # JS challenge clears itself (default 20s; 0 disables + # # the poll and reports the interstitial immediately) ``` ### Three ways to use it