diff --git a/crates/crw-renderer/src/cdp.rs b/crates/crw-renderer/src/cdp.rs index f64adc51..cf522fef 100644 --- a/crates/crw-renderer/src/cdp.rs +++ b/crates/crw-renderer/src/cdp.rs @@ -783,6 +783,54 @@ fn split_caller_headers( (ua, extra) } +/// The exact `navigator.languages` array literal inside [`STEALTH_JS`]. When a +/// request pins an exit country we swap this for the country's language list, +/// so the spoofed navigator agrees with the `Accept-Language` we send. A unit +/// test asserts the literal is still present, so editing the stealth script +/// cannot silently turn the swap into a no-op. +const STEALTH_DEFAULT_LANGUAGES: &str = "['en-US', 'en']"; + +/// The `Accept-Language` to advertise on the CDP path for the pinned exit +/// country, or `None` to leave the browser default alone. +/// +/// A caller-supplied `Accept-Language` wins: it already reaches the page +/// through `Network.setExtraHTTPHeaders`, so adding a derived value on top +/// would only contradict it. +fn language_locale( + geo_locale: Option, + extra_headers: &serde_json::Map, +) -> Option { + let locale = geo_locale?; + if extra_headers + .keys() + .any(|k| k.eq_ignore_ascii_case("accept-language")) + { + return None; + } + Some(locale) +} + +/// The `acceptLanguage` for `Network.setUserAgentOverride`, in the plain tag +/// form Chromium expects (it adds the quality weights itself). +#[cfg(test)] +fn geo_accept_language( + geo_locale: Option, + extra_headers: &serde_json::Map, +) -> Option { + language_locale(geo_locale, extra_headers).map(|l| l.cdp_accept_language()) +} + +/// Payload for `Network.setUserAgentOverride`. Without a country-derived +/// language the payload is exactly what it has always been, so the default +/// render sends byte-identical CDP traffic. +fn ua_override_params(user_agent: &str, accept_language: Option<&str>) -> serde_json::Value { + let mut params = serde_json::json!({ "userAgent": user_agent }); + if let Some(al) = accept_language { + params["acceptLanguage"] = serde_json::Value::String(al.to_string()); + } + params +} + async fn connect_chrome_with_retry_inner( name: &str, configured_ws_url: &str, @@ -2806,10 +2854,48 @@ impl CdpRenderer { .await?; } - // Inject stealth scripts before navigation so they run on every new document. + // Locale for the pinned proxy exit country. `None` (the default path, + // and any request without a `country`) leaves every CDP call below + // exactly as it was. + // + // Only the tier whose egress actually leaves from the pinned country + // composes a country-suffixed proxy login (`proxy_auth_base`), and that + // is the only tier where a country-derived locale is coherent with the + // IP. On the direct chrome tier it would pair a foreign locale with the + // box's own IP, and `Emulation.setLocaleOverride` is browser-wide in + // Chromium, so issuing it in the shared direct browser could leak into + // renders that never asked for a country. This also keeps lightpanda + // out, whose partial CDP surface may not answer `Emulation.*` at all. + // A scoped `REQUEST_PROXY` takes the egress away from the country + // credential (it wins in `fetch_inner`), so it also takes the locale + // away: the same rule `should_retry_with_default_country` applies. + let byop_active = crate::REQUEST_PROXY + .try_with(|p| p.is_some()) + .unwrap_or(false); + let geo_locale = if self.proxy_auth_base.is_some() && !byop_active { + crate::locale::request_locale() + } else { + None + }; + let (caller_ua, extra_headers) = split_caller_headers(headers); + // A caller-supplied `Accept-Language` wins over every language signal + // (header, `navigator.languages`, locale override); the clock still + // follows the exit country. + let lang_locale = language_locale(geo_locale, &extra_headers); + + // Inject stealth scripts before navigation so they run on every new + // document. With a pinned country the spoofed `navigator.languages` + // follows that country instead of announcing a US browser from a + // foreign exit IP; without one the source is the untouched constant. + let stealth_source: std::borrow::Cow<'_, str> = match lang_locale { + Some(locale) => std::borrow::Cow::Owned( + STEALTH_JS.replace(STEALTH_DEFAULT_LANGUAGES, &locale.js_languages()), + ), + None => std::borrow::Cow::Borrowed(STEALTH_JS), + }; conn.send_recv( "Page.addScriptToEvaluateOnNewDocument", - serde_json::json!({ "source": STEALTH_JS }), + serde_json::json!({ "source": stealth_source }), Some(&session_id), self.page_timeout, ) @@ -2820,8 +2906,6 @@ impl CdpRenderer { // A caller-supplied `User-Agent` wins over the tier default, the same // precedence the HTTP fetcher gives it (http_only.rs applies caller // headers last). - let (caller_ua, extra_headers) = split_caller_headers(headers); - // Present a modern UA on the CDP path too (the HTTP fetcher already does, // but renderers otherwise send the browser's own — often stale — UA, which // trips "your browser is outdated" gates). Session-scoped (so pooled @@ -2830,7 +2914,12 @@ impl CdpRenderer { // lightpanda rejects "Mozilla" UAs (→ `lightpanda_safe_ua`); it routes // Network.* → Emulation.* internally, so the method name is fine. Skip if empty. let effective_ua = caller_ua.as_deref().unwrap_or(&self.user_agent); - if !effective_ua.is_empty() { + // With an empty UA the override is still sent when a country-derived + // language exists: Chromium treats an empty `userAgent` as "keep the + // browser's own", so the language lands without touching the UA, and + // the header stays in step with the script and the locale override. + let accept_language = lang_locale.map(|l| l.cdp_accept_language()); + if !effective_ua.is_empty() || accept_language.is_some() { let ua: &str = if self.name == "lightpanda" { lightpanda_safe_ua(effective_ua) } else { @@ -2838,7 +2927,7 @@ impl CdpRenderer { }; conn.send_recv( "Network.setUserAgentOverride", - serde_json::json!({ "userAgent": ua }), + ua_override_params(ua, accept_language.as_deref()), Some(&session_id), self.page_timeout, ) @@ -2846,6 +2935,40 @@ impl CdpRenderer { .ok(); } + // Match the clock and the JS locale to the exit country as well. An + // IP/locale mismatch is a stronger bot signal than a flagged IP, so a + // German exit reporting a UTC clock and `navigator.language = en-US` + // was working against the reason the country was pinned in the first + // place. Best-effort like the two calls above: a tier that does not + // implement `Emulation.*` (lightpanda) must not fail an otherwise-fine + // render. + let overrides = geo_locale + .map(|l| { + ( + "Emulation.setTimezoneOverride", + serde_json::json!({ "timezoneId": l.timezone }), + ) + }) + .into_iter() + .chain(lang_locale.map(|l| { + ( + "Emulation.setLocaleOverride", + serde_json::json!({ "locale": l.primary_tag() }), + ) + })); + for (method, params) in overrides { + // Best-effort, but not silent: Blink backs both overrides with + // process-wide controllers and refuses a second one in the same + // renderer process, so a refusal here is the signal that two + // country-pinned renders shared a process. + if let Err(e) = conn + .send_recv(method, params, Some(&session_id), self.page_timeout) + .await + { + tracing::debug!(renderer = %self.name, method, "locale override refused: {e}"); + } + } + // Forward the caller's custom request headers. These were dropped on the // CDP path entirely (only the HTTP tier honored them), so a documented // `headers` field silently did nothing on any browser render. Additive: @@ -3642,8 +3765,10 @@ fn is_spa_text_ready(text_len: i64) -> bool { #[cfg(test)] mod tests { use super::{ - CdpRenderer, CrwError, build_auth_response, is_content_stable, is_proxy_tunnel_error, + CdpRenderer, CrwError, STEALTH_DEFAULT_LANGUAGES, STEALTH_JS, build_auth_response, + geo_accept_language, is_content_stable, is_proxy_tunnel_error, language_locale, lightpanda_safe_ua, outbound_block_label, screenshot_clip, split_caller_headers, + ua_override_params, }; use std::collections::HashMap; use std::time::Duration; @@ -3931,6 +4056,84 @@ mod tests { assert_eq!(lightpanda_safe_ua("Chrome/150.0.0.0"), "Chrome/150.0.0.0"); } + #[test] + fn stealth_js_carries_the_default_languages_literal() { + // The country path substitutes this exact substring. If a stealth-script + // edit changes the spacing or quoting, the substitution silently becomes + // a no-op, so pin it here. + assert_eq!( + STEALTH_JS.matches(STEALTH_DEFAULT_LANGUAGES).count(), + 1, + "STEALTH_JS must contain exactly one {STEALTH_DEFAULT_LANGUAGES} literal" + ); + } + + #[test] + fn ua_override_params_without_locale() { + // No country -> the payload is byte-identical to the pre-change one. + let p = ua_override_params("MyAgent/1.0", None); + assert_eq!(p, serde_json::json!({ "userAgent": "MyAgent/1.0" })); + assert!(p.get("acceptLanguage").is_none()); + } + + #[test] + fn ua_override_params_with_locale() { + let p = ua_override_params("MyAgent/1.0", Some("de-DE,de,en")); + assert_eq!(p["userAgent"], "MyAgent/1.0"); + assert_eq!(p["acceptLanguage"], "de-DE,de,en"); + } + + #[test] + fn language_locale_yields_to_caller_header() { + let de = crate::locale::locale_for_country("de"); + let empty = serde_json::Map::new(); + assert_eq!(language_locale(de, &empty), de); + assert_eq!(language_locale(None, &empty), None); + let mut caller = serde_json::Map::new(); + caller.insert( + "Accept-Language".to_string(), + serde_json::Value::String("fr-FR".to_string()), + ); + assert_eq!(language_locale(de, &caller), None); + } + + #[test] + fn geo_accept_language_yields_to_caller_header() { + let de = crate::locale::locale_for_country("de"); + let empty = serde_json::Map::new(); + + // No locale resolved (no country, or the lightpanda tier) -> nothing + // derived, whatever the headers are. + assert!(geo_accept_language(None, &empty).is_none()); + + assert_eq!( + geo_accept_language(de, &empty).as_deref(), + Some("de-DE,de,en"), + "a pinned country must drive the header, in plain tag form" + ); + + // A caller who set the header keeps it, match is case-insensitive. + for name in ["accept-language", "Accept-Language"] { + let mut caller = serde_json::Map::new(); + caller.insert( + name.to_string(), + serde_json::Value::String("fr-FR".to_string()), + ); + assert!(geo_accept_language(de, &caller).is_none(), "{name}"); + } + + // An unrelated caller header does not suppress it. + let mut caller = serde_json::Map::new(); + caller.insert( + "X-Probe".to_string(), + serde_json::Value::String("v".to_string()), + ); + assert_eq!( + geo_accept_language(de, &caller).as_deref(), + Some("de-DE,de,en") + ); + } + #[test] fn split_caller_headers_pulls_out_user_agent() { // A caller User-Agent must go to setUserAgentOverride, not into the diff --git a/crates/crw-renderer/src/lib.rs b/crates/crw-renderer/src/lib.rs index 23769695..d21a4c0c 100644 --- a/crates/crw-renderer/src/lib.rs +++ b/crates/crw-renderer/src/lib.rs @@ -54,6 +54,7 @@ pub mod egress; pub mod health_telemetry; pub mod host_limiter; pub mod http_only; +pub mod locale; pub mod preference; pub mod traits; diff --git a/crates/crw-renderer/src/locale.rs b/crates/crw-renderer/src/locale.rs new file mode 100644 index 00000000..60ef0e14 --- /dev/null +++ b/crates/crw-renderer/src/locale.rs @@ -0,0 +1,254 @@ +//! Country to locale alignment for the residential proxy exit. +//! +//! When a request pins a proxy exit country (`ScrapeRequest.country` -> +//! [`crate::REQUEST_COUNTRY`]), the browser must not keep advertising a US +//! locale and a UTC clock: an IP/locale mismatch is a stronger bot signal than +//! a flagged IP on its own. This module maps a country code to the +//! `Accept-Language` and IANA timezone a real visitor from there would present. +//! +//! Purely additive: no country (or an unknown one) yields `None` and every +//! caller keeps its previous behaviour byte for byte. + +/// The locale a visitor from one country presents. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Locale { + /// Full `Accept-Language` header value. + pub accept_language: &'static str, + /// IANA timezone id for `Emulation.setTimezoneOverride`. + pub timezone: &'static str, +} + +impl Locale { + /// The primary language tag, e.g. `de-DE`, for `Emulation.setLocaleOverride`. + pub fn primary_tag(&self) -> &'static str { + self.accept_language + .split(',') + .next() + .unwrap_or(self.accept_language) + } + + /// The language tags without quality weights: `de-DE,de;q=0.9,en;q=0.8` + /// becomes `de-DE`, `de`, `en`. + fn tags(&self) -> impl Iterator { + self.accept_language.split(',').filter_map(|part| { + let tag = part.split(';').next().unwrap_or(part).trim(); + (!tag.is_empty()).then_some(tag) + }) + } + + /// The value for CDP's `Network.setUserAgentOverride.acceptLanguage`: + /// a plain comma-separated tag list, `de-DE,de,en`. Chromium adds its own + /// quality weights to that list, so handing it the weighted header would + /// produce `de-DE,de;q=0.9;q=0.9,...`, a malformed and fingerprintable + /// header. + pub fn cdp_accept_language(&self) -> String { + self.tags().collect::>().join(",") + } + + /// The value for the stealth script's `navigator.languages`, as a JS array + /// literal: `de-DE,de;q=0.9,en;q=0.8` becomes `['de-DE', 'de', 'en']`. + pub fn js_languages(&self) -> String { + let tags: Vec = self.tags().map(|tag| format!("'{tag}'")).collect(); + format!("[{}]", tags.join(", ")) + } +} + +/// Country code (lowercase alpha-2) to locale. +/// +/// One language and one timezone per country: the capital's zone, or the most +/// populous zone for a country that spans several. The row set is the +/// countries we expect to see requested, not a provider's exit list. +/// +/// ponytail: a flat table and a linear scan. The known ceilings are one zone +/// per country (wrong for a US west-coast or a Siberian exit) and one language +/// per country (wrong for Quebec, Catalonia, Wallonia). Upgrade path if that +/// ever matters: key on the proxy's reported city rather than its country, and +/// return a list of candidate locales to pick from. +const LOCALES: &[(&str, Locale)] = &[ + ( + "ar", + loc("es-AR,es;q=0.9,en;q=0.8", "America/Argentina/Buenos_Aires"), + ), + ("at", loc("de-AT,de;q=0.9,en;q=0.8", "Europe/Vienna")), + ("au", loc("en-AU,en;q=0.9", "Australia/Sydney")), + ("be", loc("nl-BE,nl;q=0.9,en;q=0.8", "Europe/Brussels")), + ("br", loc("pt-BR,pt;q=0.9,en;q=0.8", "America/Sao_Paulo")), + ("ca", loc("en-CA,en;q=0.9", "America/Toronto")), + ("ch", loc("de-CH,de;q=0.9,en;q=0.8", "Europe/Zurich")), + ("cn", loc("zh-CN,zh;q=0.9,en;q=0.8", "Asia/Shanghai")), + ("cz", loc("cs-CZ,cs;q=0.9,en;q=0.8", "Europe/Prague")), + ("de", loc("de-DE,de;q=0.9,en;q=0.8", "Europe/Berlin")), + ("dk", loc("da-DK,da;q=0.9,en;q=0.8", "Europe/Copenhagen")), + ("es", loc("es-ES,es;q=0.9,en;q=0.8", "Europe/Madrid")), + ("fi", loc("fi-FI,fi;q=0.9,en;q=0.8", "Europe/Helsinki")), + ("fr", loc("fr-FR,fr;q=0.9,en;q=0.8", "Europe/Paris")), + ("gb", loc("en-GB,en;q=0.9", "Europe/London")), + ("ie", loc("en-IE,en;q=0.9", "Europe/Dublin")), + ("in", loc("en-IN,en;q=0.9", "Asia/Kolkata")), + ("it", loc("it-IT,it;q=0.9,en;q=0.8", "Europe/Rome")), + ("jp", loc("ja-JP,ja;q=0.9,en;q=0.8", "Asia/Tokyo")), + ("kr", loc("ko-KR,ko;q=0.9,en;q=0.8", "Asia/Seoul")), + ("mx", loc("es-MX,es;q=0.9,en;q=0.8", "America/Mexico_City")), + ("nl", loc("nl-NL,nl;q=0.9,en;q=0.8", "Europe/Amsterdam")), + ("no", loc("nb-NO,nb;q=0.9,en;q=0.8", "Europe/Oslo")), + ("nz", loc("en-NZ,en;q=0.9", "Pacific/Auckland")), + ("pl", loc("pl-PL,pl;q=0.9,en;q=0.8", "Europe/Warsaw")), + ("pt", loc("pt-PT,pt;q=0.9,en;q=0.8", "Europe/Lisbon")), + ("ru", loc("ru-RU,ru;q=0.9,en;q=0.8", "Europe/Moscow")), + ("se", loc("sv-SE,sv;q=0.9,en;q=0.8", "Europe/Stockholm")), + ("tr", loc("tr-TR,tr;q=0.9,en;q=0.8", "Europe/Istanbul")), + ("ua", loc("uk-UA,uk;q=0.9,en;q=0.8", "Europe/Kyiv")), + // Not an ISO code, but the proxy credential passes it through untouched + // and providers commonly treat it as an alias of GB. + ("uk", loc("en-GB,en;q=0.9", "Europe/London")), + ("us", loc("en-US,en;q=0.9", "America/New_York")), +]; + +const fn loc(accept_language: &'static str, timezone: &'static str) -> Locale { + Locale { + accept_language, + timezone, + } +} + +/// Look up the locale for an ISO 3166-1 alpha-2 country code. +/// +/// Accepts any case and surrounding whitespace, the same normalization the +/// proxy-credential composition applies. Returns `None` for a malformed code +/// or a country we have no row for, which keeps every caller on its default. +pub fn locale_for_country(country: &str) -> Option { + let cc = country.trim(); + if cc.len() != 2 || !cc.chars().all(|c| c.is_ascii_alphabetic()) { + return None; + } + LOCALES + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(cc)) + .map(|(_, locale)| *locale) +} + +/// The locale for the country pinned on the current request, if any. +/// +/// ponytail: reads only [`crate::REQUEST_COUNTRY`], not +/// `renderer.proxy_default_country`. The HTTP fetcher has no handle on the +/// renderer config and threading one through its three constructors buys +/// nothing today, since no deployment sets a default country. One asymmetry +/// follows: the CDP tier's country-fallback retry re-scopes `REQUEST_COUNTRY` +/// to that default before it re-runs, so only the retry attempt carries the +/// default country's locale. Upgrade path: resolve request country -> default +/// once where `REQUEST_COUNTRY` is scoped, so every reader sees the same value. +pub fn request_locale() -> Option { + crate::REQUEST_COUNTRY + .try_with(|c| c.clone()) + .ok() + .flatten() + .as_deref() + .and_then(locale_for_country) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn locale_table_is_well_formed() { + let mut seen: Vec<&str> = Vec::new(); + for (cc, locale) in LOCALES { + assert_eq!(cc.len(), 2, "{cc}: key must be alpha-2"); + assert!( + cc.chars().all(|c| c.is_ascii_lowercase()), + "{cc}: key must be lowercase" + ); + assert!(!seen.contains(cc), "{cc}: duplicate row"); + seen.push(cc); + + // First tag is a region-qualified language, e.g. "de-DE". + let primary = locale.primary_tag(); + let (lang, region) = primary + .split_once('-') + .unwrap_or_else(|| panic!("{primary}: needs a region")); + assert_eq!(lang.len(), 2, "{primary}: language subtag"); + // "uk" is the one alias row: it carries GB's locale. + let expected_region = if *cc == "uk" { "gb" } else { cc }; + assert!( + region.eq_ignore_ascii_case(expected_region), + "{primary}: region must match the row key {cc}" + ); + + // Timezone is an IANA "Area/Location" id. + assert!( + locale.timezone.contains('/') && !locale.timezone.contains(' '), + "{}: not an IANA zone id", + locale.timezone + ); + } + assert!(seen.len() >= 30, "table shrank unexpectedly"); + } + + #[test] + fn locale_lookup_normalizes_input() { + let berlin = "Europe/Berlin"; + assert_eq!(locale_for_country("de").unwrap().timezone, berlin); + assert_eq!(locale_for_country("DE").unwrap().timezone, berlin); + assert_eq!(locale_for_country(" de ").unwrap().timezone, berlin); + // Malformed or unknown codes must not resolve. + assert!(locale_for_country("xx").is_none()); + assert!(locale_for_country("").is_none()); + assert!(locale_for_country("deu").is_none()); + assert!(locale_for_country("d1").is_none()); + } + + #[test] + fn us_locale_matches_the_legacy_constant() { + // The pre-existing hardcoded header value. `country=us` must therefore + // change nothing but the clock. + assert_eq!( + locale_for_country("us").unwrap().accept_language, + "en-US,en;q=0.9" + ); + } + + #[test] + fn cdp_accept_language_is_a_plain_tag_list() { + let de = locale_for_country("de").unwrap(); + assert_eq!(de.cdp_accept_language(), "de-DE,de,en"); + let us = locale_for_country("us").unwrap(); + assert_eq!(us.cdp_accept_language(), "en-US,en"); + } + + #[test] + fn js_languages_drops_q_values() { + let de = locale_for_country("de").unwrap(); + assert_eq!(de.js_languages(), "['de-DE', 'de', 'en']"); + assert_eq!(de.primary_tag(), "de-DE"); + + let us = locale_for_country("us").unwrap(); + assert_eq!(us.js_languages(), "['en-US', 'en']"); + } + + #[tokio::test] + async fn request_locale_reads_the_task_local() { + // Outside any scope the task-local is unset. + assert!(request_locale().is_none()); + + crate::REQUEST_COUNTRY + .scope(Some("fr".to_string()), async { + assert_eq!(request_locale().unwrap().timezone, "Europe/Paris"); + }) + .await; + + // A scoped-but-empty country stays on the default path. + crate::REQUEST_COUNTRY + .scope(None, async { + assert!(request_locale().is_none()); + }) + .await; + + // An unknown country stays on the default path too. + crate::REQUEST_COUNTRY + .scope(Some("zz".to_string()), async { + assert!(request_locale().is_none()); + }) + .await; + } +}