Repository navigation
Expand file tree
/
Copy pathclient.rs
More file actions
7220 lines (6858 loc) · 282 KB
/
Copy pathclient.rs
File metadata and controls
7220 lines (6858 loc) · 282 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! HTTP client for the resolved provider route.
//!
//! Routes reach the provider through its OpenAI-compatible or native wire
//! surface; `/chat/completions` is the common primary endpoint.
use std::collections::HashMap;
#[cfg(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
))]
use std::sync::Mutex as StdMutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use anyhow::{Context, Result, bail};
use base64::{Engine as _, engine::general_purpose};
use futures_util::StreamExt;
use reqwest::header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use tokio::sync::{
Mutex as AsyncMutex, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, OwnedSemaphorePermit, RwLock,
Semaphore,
};
use codewhale_config::catalog::{
CatalogOffering, CatalogRefreshError, CatalogSource, ProviderCatalogDelta,
base_url_fingerprint, now_unix,
};
#[cfg(any(test, feature = "test-support"))]
use codewhale_config::catalog::{CatalogSnapshot, CatalogStatus, ProviderCatalogCache};
use codewhale_config::provider::WireFormat;
use codewhale_config::route::{
LogicalModelRef, ReadyRouteCandidate, RouteLimits, RouteRequest, RouteResolver,
};
use codewhale_config::{auth_mode_disables_api_key, is_upstream_auth_header};
use crate::llm_client::{
LlmClient, LlmError, OutboundWireObservation, RetryConfig as LlmRetryConfig,
ToolWireMeasurement, extract_retry_after, observe_outbound_wire, retry_delay_from_error_body,
sanitize_http_error_body, with_retry,
};
use codewhale_config::ProviderKind;
use codewhale_runtime::config::{
Config, ProviderIdentity, RetryPolicy, validate_route, wire_model_for_provider_route,
};
#[cfg(test)]
#[path = "client/catalog_tests.rs"]
mod catalog_tests;
use codewhale_models::Role;
use codewhale_models::{
ContentBlock, Message, MessageRequest, MessageResponse, SystemPrompt, Usage,
};
use codewhale_runtime::logging;
use codewhale_secrets::sanitize::redact_url_for_display;
/// Every provider request that can feed the interactive TUI's attached CWC run
/// takes a shared permit at this lowest common dispatch seam. Runtime Chat holds
/// the exclusive permit from native admission through durable terminal
/// acknowledgement. This covers ordinary turns, auto-route classification,
/// advisor calls, detached subagents, compaction, purge, and streaming without
/// serializing independent RuntimeThreadManager stores.
#[cfg(not(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
)))]
static RUNTIME_CHAT_INFERENCE_GATE: OnceLock<Arc<RwLock<()>>> = OnceLock::new();
/// Unit tests run many independent Tokio runtimes in one process. Keying the
/// otherwise-identical gate by runtime keeps unrelated libtest cases from
/// manufacturing contention while preserving exact read/write behavior among
/// tasks on the same current-thread or multi-thread runtime.
#[cfg(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
))]
static RUNTIME_CHAT_INFERENCE_TEST_GATES: OnceLock<
StdMutex<HashMap<RuntimeChatInferenceTestScope, std::sync::Weak<RwLock<()>>>>,
> = OnceLock::new();
#[cfg(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
))]
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
enum RuntimeChatInferenceTestScope {
Runtime(tokio::runtime::Id),
Thread(std::thread::ThreadId),
}
#[cfg(not(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
)))]
fn runtime_chat_inference_gate() -> Arc<RwLock<()>> {
Arc::clone(RUNTIME_CHAT_INFERENCE_GATE.get_or_init(|| Arc::new(RwLock::new(()))))
}
#[cfg(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
))]
fn runtime_chat_inference_gate() -> Arc<RwLock<()>> {
let scope = tokio::runtime::Handle::try_current()
.map(|handle| RuntimeChatInferenceTestScope::Runtime(handle.id()))
.unwrap_or_else(|_| RuntimeChatInferenceTestScope::Thread(std::thread::current().id()));
let mut gates = RUNTIME_CHAT_INFERENCE_TEST_GATES
.get_or_init(|| StdMutex::new(HashMap::new()))
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(gate) = gates.get(&scope).and_then(std::sync::Weak::upgrade) {
return gate;
}
let gate = Arc::new(RwLock::new(()));
gates.insert(scope, Arc::downgrade(&gate));
gate
}
/// Exclusive ownership retained by isolated Runtime Chat.
pub struct RuntimeChatInferenceOwnership {
_gate: OwnedRwLockWriteGuard<()>,
}
/// Shared participation retained for the full provider-output lifecycle.
pub struct RemoteControlInferencePermit {
_gate: OwnedRwLockReadGuard<()>,
}
#[cfg(any(
test,
feature = "test-support",
all(feature = "test-thread-scoped-state", debug_assertions)
))]
pub async fn acquire_runtime_chat_inference_ownership() -> RuntimeChatInferenceOwnership {
let gate = runtime_chat_inference_gate().write_owned().await;
RuntimeChatInferenceOwnership { _gate: gate }
}
pub fn try_acquire_runtime_chat_inference_ownership() -> Option<RuntimeChatInferenceOwnership> {
let gate = runtime_chat_inference_gate().try_write_owned().ok()?;
Some(RuntimeChatInferenceOwnership { _gate: gate })
}
/// Join the attached interactive CWC run as a provider-output participant.
/// Standalone background adapters that bypass `CodewhaleClient::create_message`
/// use this guard and retain it through response decoding.
pub async fn acquire_remote_control_inference_participant() -> RemoteControlInferencePermit {
let gate = runtime_chat_inference_gate().read_owned().await;
RemoteControlInferencePermit { _gate: gate }
}
// === Types ===
/// Model descriptor returned by the provider's `/v1/models` endpoint.
#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub struct AvailableModel {
pub id: String,
pub owned_by: Option<String>,
pub created: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub display_name: Option<String>,
}
/// Request payload for Xiaomi MiMo speech synthesis models.
///
/// MiMo-V2.5-TTS / MiMo-V2-TTS use the OpenAI-compatible
/// `/v1/chat/completions` endpoint: the optional style/voice instruction is
/// sent as a `user` message, while the text to synthesize is sent as an
/// `assistant` message.
#[derive(Debug, Clone)]
pub struct SpeechSynthesisRequest {
pub model: String,
pub text: String,
pub instruction: Option<String>,
pub audio_format: String,
pub voice: Option<String>,
}
/// Decoded speech synthesis result.
#[derive(Debug, Clone)]
pub struct SpeechSynthesisResponse {
pub model: String,
pub audio_format: String,
pub audio_bytes: Vec<u8>,
pub transcript: Option<String>,
pub voice: Option<String>,
}
/// One decoded provider response from the auxiliary translation path.
///
/// The immutable route and provider-reported usage travel beside the semantic
/// translation result so callers can account for a successful provider call
/// before rejecting an incomplete, empty, or otherwise unusable translation.
/// `usage == None` is distinct from a transport failure: the provider returned
/// a response, but omitted the receipt needed to price it exactly.
pub struct TranslationProviderResponse {
pub translated: Result<String>,
pub route: crate::cost_status::EffectiveRouteEnvelope,
pub usage: Option<Usage>,
pub accounting: Option<crate::cost_status::RuntimeUsageRecord>,
}
/// Universal client for the resolved provider route.
#[must_use]
pub struct CodewhaleClient {
pub(super) http_client: reqwest::Client,
// Catalogs and probes must never forward frozen custom auth headers to a
// provider-supplied redirect destination. Inference keeps its own policy.
models_http_client: reqwest::Client,
/// HTTP/1.1-only twin of [`Self::http_client`], used for automatic
/// stream-header fallback when H2 stalls. Same auth and headers.
pub(super) http1_client: reqwest::Client,
api_key: String,
/// Where `api_key` came from (secret store slot, config file, env var
/// name, CLI, OAuth, …), named in authentication errors (#6528).
api_key_source: String,
/// For a subscription sign-in route (ChatGPT, xAI OAuth): which account
/// is signed in and how to switch, appended to plan-quota errors. Holds
/// an account label only, never token material.
subscription_limit_guidance: Option<String>,
/// Reviewed runtime authority and OAuth descriptor travel with the frozen route.
plugin_provider: Option<Box<codewhale_runtime::config::ProviderConfig>>,
/// Read-only diagnostic probes must never migrate or refresh the grant.
plugin_oauth_read_only: bool,
session_provider_keys_only: bool,
claude_oauth_config: Option<Box<Config>>,
/// Exact configured credential values removed from model-bound tool
/// results. Structural redaction handles config/JSON assignments, while
/// this list closes the gap for bare provider tokens with no recognizable
/// prefix (for example token-plan and provider-specific keys).
model_bound_secret_values: Arc<Vec<String>>,
/// Exact values a catalog endpoint could echo back at us: the active API
/// key and every user-configured HTTP header value, plus everything in
/// `model_bound_secret_values`.
///
/// Deliberately a second, wider list rather than a widening of that one:
/// they answer different questions at different trust boundaries. That
/// list is "what must never reach a *model*", which is why it covers only
/// auth-shaped headers and values of at least
/// `MIN_EXACT_SECRET_CHARS`. This one is "what must never come back out
/// of an endpoint the user typed during setup" (#6173), where a
/// three-character key is still a key and a custom header the user
/// configured is still theirs.
catalog_error_secret_values: Arc<Vec<String>>,
/// Whether credential-shaped tool output is masked before it is sent to an
/// upstream model. The safe default is `true`; it is `false` only after the
/// user disabled `[redaction] model_bound` and confirmed the opt-out on the
/// startup gate (see [`codewhale_config::redaction`]). Routing/classification
/// summaries and durable goal-state text keep their own always-on redaction
/// regardless of this flag.
model_bound_masking: bool,
pub(super) base_url: String,
pub(super) api_provider: ProviderKind,
/// Exact configured provider identity and billing mode frozen when this
/// client is built. Child/tool calls only carry the client at dispatch, so
/// these route facts must travel with it instead of being reconstructed
/// from the mutable parent session at completion time.
admitted_identity: ProviderIdentity,
openrouter_vendor: Option<String>,
billing_surface: Option<String>,
billing_mode: crate::cost_status::RouteBillingMode,
configured_models: Arc<Vec<codewhale_config::catalog::configured::ConfiguredModel>>,
context_window_override: Option<u32>,
model_context_windows: Option<std::collections::BTreeMap<String, u32>>,
/// Non-secret limits frozen from the same resolved candidate as the
/// endpoint and wire model. Auxiliary calls carry only this client, so
/// they must not reconstruct output caps with `None` and discard a custom
/// route's context window.
route_limits: Option<RouteLimits>,
context_window: Option<crate::route_runtime::ContextWindowResolution>,
route_facts: Option<Arc<codewhale_config::catalog::CatalogOffering>>,
/// Verified ChatGPT subject captured with Codewhale's own selected grant.
pub(super) codex_account_id: Option<String>,
/// Opaque reasoning is bound to this verified grant identity, stable across
/// token refresh and absent on custom API-key routes.
pub(super) chatgpt_reasoning_api: Option<String>,
wire_format: WireFormat,
retry: RetryPolicy,
/// Auxiliary inspection calls use the normal bounded retry schedule but
/// never publish retry/rate-limit state into process-global UI cells.
isolated_request_state: bool,
/// Whether this concrete client can contribute provider output to the
/// interactive TUI's attached CWC run. Isolated Runtime Chat executes under
/// the host's exclusive permit; independent RuntimeThreadManager stores do
/// not share that run and remain concurrent.
remote_control_inference_participant: bool,
default_model: String,
connection_health: Arc<AsyncMutex<ConnectionHealth>>,
rate_limiter: Arc<AsyncMutex<TokenBucket>>,
request_concurrency: Option<ProviderConcurrencyLimiter>,
path_suffix: Option<String>,
/// Unit tests keep the semantic route exact while sending the actual
/// production request through a local capture server. This field is
/// compiled out of release builds.
#[cfg(any(test, feature = "test-support"))]
test_chat_transport_base_url: Option<String>,
/// Messages equivalent of `test_chat_transport_base_url`; keeps exact
/// route shaping bound to the semantic endpoint while tests capture on a
/// local server.
#[cfg(any(test, feature = "test-support"))]
test_messages_transport_base_url: Option<String>,
pub(super) reasoning_stream_style: Option<String>,
pub(super) stream_idle_timeout: Duration,
/// Bounded wait for SSE response headers, resolved once from
/// `[stream].open_timeout_secs`, legacy `[tui]`, or the environment fallback.
pub(super) stream_open_timeout: Duration,
/// HTTP/1.1 pin resolved once from `Config::force_http1` (#6700); the
/// single source every client builder and stream open reads.
pub(super) force_http1: bool,
}
const CONNECTION_FAILURE_THRESHOLD: u32 = 2;
const RECOVERY_PROBE_COOLDOWN: Duration = Duration::from_secs(15);
const DEFAULT_CLIENT_RATE_LIMIT_RPS: f64 = 8.0;
const DEFAULT_CLIENT_RATE_LIMIT_BURST: f64 = 16.0;
const ALLOW_INSECURE_HTTP_ENV: &str = "CODEWHALE_ALLOW_INSECURE_HTTP";
/// Legacy alias for [`ALLOW_INSECURE_HTTP_ENV`].
const LEGACY_ALLOW_INSECURE_HTTP_ENV: &str = "DEEPSEEK_ALLOW_INSECURE_HTTP";
fn client_user_agent(_api_provider: ProviderKind) -> &'static str {
concat!(
"Mozilla/5.0 (compatible; codewhale/",
env!("CARGO_PKG_VERSION"),
"; +https://github.com/codewhale-hq/CodeWhale)"
)
}
/// Upper bound on a single sleep inside the provider-wide rate-limit pause
/// loop in `send_with_retry`. The pause window lives in process-global state
/// (`retry_status`), so waiting requests re-poll it on this cadence instead
/// of committing to the full remaining window up front.
const RATE_LIMIT_PAUSE_RECHECK_INTERVAL: Duration = Duration::from_millis(250);
/// Total budget for one non-streaming request. Two layers use it: each
/// attempt carries it as a reqwest per-request total (connect through body
/// end, so a trickling body cannot extend forever), and the retry loop
/// through `send_with_retry` is wrapped in one outer envelope of the same
/// length (all attempts, backoff, and honored Retry-After included). The
/// shared client intentionally has no client-level total timeout, so without
/// these nothing bounds a non-streaming completion: a provider that accepts
/// the connection and then stalls — or a gateway answering 429 +
/// `Retry-After: 3600` forever — wedged the caller indefinitely.
///
/// Streaming paths never carry it: their opens go through
/// `send_stream_open_with_retry`, which sets no per-request total (a total
/// would ride on the returned body and hard-cut a live stream), so a stream
/// stays bounded by its open cap and per-chunk idle checks only.
pub(super) const NON_STREAMING_REQUEST_ENVELOPE: Duration = Duration::from_secs(1800);
#[cfg(any(test, feature = "test-support"))]
static TEST_NON_STREAMING_ENVELOPE_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
fn non_streaming_request_envelope() -> Duration {
#[cfg(any(test, feature = "test-support"))]
{
let ms = TEST_NON_STREAMING_ENVELOPE_MS.load(std::sync::atomic::Ordering::SeqCst);
if ms > 0 {
return Duration::from_millis(ms);
}
}
NON_STREAMING_REQUEST_ENVELOPE
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ConnectionState {
Healthy,
Degraded,
Recovering,
}
#[derive(Debug)]
struct ConnectionHealth {
state: ConnectionState,
consecutive_failures: u32,
last_failure: Option<Instant>,
last_success: Option<Instant>,
last_probe: Option<Instant>,
}
impl Default for ConnectionHealth {
fn default() -> Self {
Self {
state: ConnectionState::Healthy,
consecutive_failures: 0,
last_failure: None,
last_success: None,
last_probe: None,
}
}
}
#[derive(Debug)]
struct TokenBucket {
enabled: bool,
capacity: f64,
tokens: f64,
refill_per_sec: f64,
last_refill: Instant,
}
#[derive(Debug, Clone)]
struct ProviderConcurrencyLimiter {
semaphore: Arc<Semaphore>,
active: Arc<AtomicUsize>,
limit: usize,
}
struct ProviderRequestPermit {
_permit: OwnedSemaphorePermit,
active: Arc<AtomicUsize>,
}
impl ProviderConcurrencyLimiter {
fn new(limit: usize) -> Self {
let limit = limit.max(1);
Self {
semaphore: Arc::new(Semaphore::new(limit)),
active: Arc::new(AtomicUsize::new(0)),
limit,
}
}
async fn acquire(&self) -> Option<ProviderRequestPermit> {
let permit = Arc::clone(&self.semaphore).acquire_owned().await.ok()?;
self.active.fetch_add(1, Ordering::AcqRel);
Some(ProviderRequestPermit {
_permit: permit,
active: Arc::clone(&self.active),
})
}
fn active(&self) -> usize {
self.active.load(Ordering::Acquire)
}
fn limit(&self) -> usize {
self.limit
}
}
impl Drop for ProviderRequestPermit {
fn drop(&mut self) {
self.active.fetch_sub(1, Ordering::AcqRel);
}
}
impl TokenBucket {
fn from_env() -> Self {
let rps = std::env::var("CODEWHALE_RATE_LIMIT_RPS")
.or_else(|_| std::env::var("DEEPSEEK_RATE_LIMIT_RPS"))
.ok()
.and_then(|v| v.parse::<f64>().ok())
.unwrap_or(DEFAULT_CLIENT_RATE_LIMIT_RPS)
.max(0.0);
let burst = std::env::var("CODEWHALE_RATE_LIMIT_BURST")
.or_else(|_| std::env::var("DEEPSEEK_RATE_LIMIT_BURST"))
.ok()
.and_then(|v| v.parse::<f64>().ok())
.unwrap_or(DEFAULT_CLIENT_RATE_LIMIT_BURST)
.max(1.0);
let enabled = rps > 0.0;
Self {
enabled,
capacity: burst,
tokens: burst,
refill_per_sec: rps,
last_refill: Instant::now(),
}
}
fn refill(&mut self, now: Instant) {
if !self.enabled {
return;
}
let elapsed = now.duration_since(self.last_refill).as_secs_f64();
self.last_refill = now;
self.tokens = (self.tokens + elapsed * self.refill_per_sec).min(self.capacity);
}
/// Reserve `tokens` and report how long the caller must sleep first.
///
/// The debt is *kept* (the balance is allowed to go negative) rather than
/// floored at zero. Callers release the bucket lock before sleeping, so a
/// floored balance would hand every queued waiter the same short delay and
/// they would all wake — and fire — at the same instant, which is the
/// burst the configured limit exists to prevent. Carrying the deficit
/// spaces successive waiters one refill interval apart, and `refill`'s
/// clamp to `capacity` still caps how much credit an idle bucket banks.
fn delay_until_available(&mut self, tokens: f64) -> Option<Duration> {
if !self.enabled {
return None;
}
let now = Instant::now();
self.refill(now);
self.tokens -= tokens;
if self.tokens >= 0.0 {
return None;
}
if self.refill_per_sec <= 0.0 {
return Some(Duration::from_secs(1));
}
Some(Duration::from_secs_f64(-self.tokens / self.refill_per_sec))
}
}
fn apply_request_success(health: &mut ConnectionHealth, now: Instant) -> bool {
let recovered = health.state != ConnectionState::Healthy;
health.state = ConnectionState::Healthy;
health.consecutive_failures = 0;
health.last_success = Some(now);
recovered
}
fn apply_request_failure(health: &mut ConnectionHealth, now: Instant) {
health.consecutive_failures = health.consecutive_failures.saturating_add(1);
health.last_failure = Some(now);
if health.consecutive_failures >= CONNECTION_FAILURE_THRESHOLD {
health.state = ConnectionState::Degraded;
}
}
fn mark_recovery_probe_if_due(health: &mut ConnectionHealth, now: Instant) -> bool {
if health.state == ConnectionState::Healthy {
return false;
}
if health
.last_probe
.is_some_and(|last| now.duration_since(last) < RECOVERY_PROBE_COOLDOWN)
{
return false;
}
health.last_probe = Some(now);
health.state = ConnectionState::Recovering;
true
}
/// The one command that replaces a rejected key, by where it came from.
fn auth_fix_hint(route: &str, key_source: &str) -> String {
if key_source == "Claude sign-in" {
"codewhale auth claude".to_string()
} else if key_source.starts_with("--api-key") {
"pass a valid --api-key".to_string()
} else if let Some(rest) = key_source.strip_prefix("env var ")
&& rest.contains("api_key_env")
{
let name = rest.split_whitespace().next().unwrap_or(rest);
format!("set {name} to a valid key")
} else if key_source.contains("OAuth") || key_source.contains("login") {
format!("sign in again for {route}")
} else {
format!("codewhale auth set --provider {route}")
}
}
impl Clone for CodewhaleClient {
fn clone(&self) -> Self {
Self {
http_client: self.http_client.clone(),
models_http_client: self.models_http_client.clone(),
http1_client: self.http1_client.clone(),
api_key: self.api_key.clone(),
api_key_source: self.api_key_source.clone(),
subscription_limit_guidance: self.subscription_limit_guidance.clone(),
plugin_provider: self.plugin_provider.clone(),
plugin_oauth_read_only: self.plugin_oauth_read_only,
session_provider_keys_only: self.session_provider_keys_only,
claude_oauth_config: self.claude_oauth_config.clone(),
model_bound_secret_values: Arc::clone(&self.model_bound_secret_values),
catalog_error_secret_values: Arc::clone(&self.catalog_error_secret_values),
model_bound_masking: self.model_bound_masking,
base_url: self.base_url.clone(),
api_provider: self.api_provider,
admitted_identity: self.admitted_identity.clone(),
openrouter_vendor: self.openrouter_vendor.clone(),
billing_surface: self.billing_surface.clone(),
billing_mode: self.billing_mode,
configured_models: Arc::clone(&self.configured_models),
context_window_override: self.context_window_override,
model_context_windows: self.model_context_windows.clone(),
route_limits: self.route_limits,
context_window: self.context_window,
route_facts: self.route_facts.clone(),
codex_account_id: self.codex_account_id.clone(),
chatgpt_reasoning_api: self.chatgpt_reasoning_api.clone(),
wire_format: self.wire_format,
retry: self.retry.clone(),
isolated_request_state: self.isolated_request_state,
remote_control_inference_participant: self.remote_control_inference_participant,
default_model: self.default_model.clone(),
connection_health: self.connection_health.clone(),
rate_limiter: self.rate_limiter.clone(),
request_concurrency: self.request_concurrency.clone(),
path_suffix: self.path_suffix.clone(),
#[cfg(any(test, feature = "test-support"))]
test_chat_transport_base_url: self.test_chat_transport_base_url.clone(),
#[cfg(any(test, feature = "test-support"))]
test_messages_transport_base_url: self.test_messages_transport_base_url.clone(),
reasoning_stream_style: self.reasoning_stream_style.clone(),
stream_idle_timeout: self.stream_idle_timeout,
stream_open_timeout: self.stream_open_timeout,
force_http1: self.force_http1,
}
}
}
const MIN_EXACT_SECRET_CHARS: usize = 8;
pub use codewhale_config::apply_openrouter_vendor;
fn push_model_bound_secret(values: &mut Vec<String>, value: Option<&str>) {
let Some(value) = value
.map(str::trim)
.filter(|value| !value.is_empty() && value.chars().count() >= MIN_EXACT_SECRET_CHARS)
else {
return;
};
if !values.iter().any(|existing| existing == value) {
values.push(value.to_string());
}
}
fn model_bound_secret_store_slot(provider: ProviderKind) -> Option<&'static str> {
(provider != ProviderKind::Custom).then(|| provider.secret_store_slot())
}
fn push_file_backed_model_bound_secrets(values: &mut Vec<String>) {
// Unit tests must never inspect the developer's real credential store.
// The isolated regression below opts in with a temporary CODEWHALE_HOME,
// matching Config's existing secret-store test discipline.
#[cfg(any(test, feature = "test-support"))]
if !codewhale_paths::codewhale_home_is_explicit()
|| std::env::var_os("CODEWHALE_SECRET_BACKEND").is_none()
{
return;
}
// Redaction needs only a best-effort view of inactive file-backed
// credentials. It must not cause a legacy-store migration merely because a
// client is being constructed (notably for `doctor`'s live probe). Keep
// this file-only to avoid a burst of OS-keychain prompts for inactive
// providers; the active credential is already supplied by the route
// resolver.
let secrets = codewhale_secrets::Secrets::file_backed_read_only();
let mut slots = Vec::new();
for provider in codewhale_config::descriptors::provider_compatibility()
.iter()
.map(|row| row.kind)
{
let Some(slot) = model_bound_secret_store_slot(provider) else {
continue;
};
if !slots.contains(&slot) {
slots.push(slot);
}
}
// The legacy literal `provider = "custom"` route owns this durable slot.
slots.push("custom");
for slot in slots {
if let Ok(Some(secret)) = secrets.get(slot) {
push_model_bound_secret(values, Some(&secret));
}
}
}
pub fn active_model_bound_secret_values(config: &Config) -> Vec<String> {
let active_key = config.active_route_api_key_read_only().unwrap_or_default();
configured_model_bound_secret_values(config, &active_key)
}
pub fn configured_model_bound_secret_values(config: &Config, active_api_key: &str) -> Vec<String> {
let mut values = Vec::new();
push_model_bound_secret(&mut values, Some(active_api_key));
for (_, credential) in config.session_provider_keys.read().values() {
push_model_bound_secret(&mut values, Some(credential.expose_secret()));
}
push_model_bound_secret(&mut values, config.sandbox_api_key.as_deref());
push_model_bound_secret(
&mut values,
config
.search
.as_ref()
.and_then(|search| search.api_key.as_deref()),
);
push_model_bound_secret(
&mut values,
config
.vision_model
.as_ref()
.and_then(|vision| vision.api_key.as_deref()),
);
if let Some(headers) = config.http_headers.as_ref() {
for (name, value) in headers {
if is_upstream_auth_header(name) {
push_model_bound_secret(&mut values, Some(value));
}
}
}
for row in codewhale_config::descriptors::provider_compatibility()
.iter()
.filter(|row| row.kind != ProviderKind::Custom)
{
for env_name in row.kind.provider().env_vars() {
if let Ok(value) = std::env::var(env_name) {
push_model_bound_secret(&mut values, Some(&value));
}
}
let Some(provider_config) = config
.providers
.as_ref()
.and_then(|tables| codewhale_config::provider_config_table!(@read tables, row.id))
else {
continue;
};
push_model_bound_secret(&mut values, provider_config.api_key.as_deref());
if let Some(headers) = provider_config.http_headers.as_ref() {
for (name, value) in headers {
if is_upstream_auth_header(name) {
push_model_bound_secret(&mut values, Some(value));
}
}
}
}
if let Some(providers) = config.providers.as_ref() {
for provider_config in providers.custom.values() {
push_model_bound_secret(&mut values, provider_config.api_key.as_deref());
if let Some(env_name) = provider_config
.api_key_env
.as_deref()
.map(str::trim)
.filter(|name| !name.is_empty())
&& let Ok(value) = std::env::var(env_name)
{
push_model_bound_secret(&mut values, Some(&value));
}
if let Some(headers) = provider_config.http_headers.as_ref() {
for (name, value) in headers {
if is_upstream_auth_header(name) {
push_model_bound_secret(&mut values, Some(value));
}
}
}
}
}
// The decision router's TypeSafe key is no chat provider's key; its env
// form must still never reach a model. (`for_decision_route` adds the key
// from every source to its own client.)
if let Ok(value) = std::env::var(system_one::TYPESAFE_API_KEY_ENV) {
push_model_bound_secret(&mut values, Some(&value));
}
push_file_backed_model_bound_secrets(&mut values);
// Replace longer values first in case one credential happens to contain
// another as a prefix.
values.sort_by_key(|value| std::cmp::Reverse(value.len()));
values
}
/// Everything a catalog probe could have sent that must not come back.
///
/// Longest first, so a value that contains another is masked whole.
fn catalog_error_secret_values(
active_api_key: &str,
http_headers: &HashMap<String, String>,
model_bound: &[String],
) -> Vec<String> {
let mut values: Vec<String> = Vec::new();
let mut push = |value: &str| {
let value = value.trim();
if !value.is_empty() && !values.iter().any(|existing| existing == value) {
values.push(value.to_string());
}
};
// No length floor here, unlike `push_model_bound_secret`: a short key
// echoed back by an untrusted endpoint is still a leaked key. The cost of
// being wrong is a suppressed message, which is what this path did before.
push(active_api_key);
for value in http_headers.values() {
push(value);
}
for value in model_bound {
push(value);
}
values.sort_by_key(|value| std::cmp::Reverse(value.len()));
values
}
/// The opaque values in a request's URL query — the pagination cursor.
///
/// Collected in both forms: decoded, as a provider that parsed the cursor
/// would echo it, and raw, as one that quoted the URL back would.
fn request_query_secret_values(url: &reqwest::Url) -> Vec<String> {
let mut values: Vec<String> = Vec::new();
let mut push = |value: String| {
if !value.trim().is_empty() && !values.contains(&value) {
values.push(value);
}
};
for (_, value) in url.query_pairs() {
push(value.into_owned());
}
for pair in url.query().unwrap_or_default().split('&') {
if let Some((_, value)) = pair.split_once('=') {
push(value.to_string());
}
}
values.sort_by_key(|value| std::cmp::Reverse(value.len()));
values
}
pub fn redact_model_bound_text(text: &str, exact_secret_values: &[String]) -> String {
let mut redacted = text.to_string();
for secret in exact_secret_values {
redacted = redacted.replace(secret, codewhale_config::persistence::REDACTED);
}
// Tool results feed exact-match edits, so only credential-shaped values
// are masked here; key-only hits (`password: credentials?.password`) stay
// byte-exact. Logs and previews keep the broad key-based scrubber.
codewhale_config::persistence::redact_model_bound_secrets(&redacted)
}
pub fn redact_json_model_bound_text(
value: &serde_json::Value,
secrets: &[String],
) -> serde_json::Value {
fn mask_exact_values(value: &mut serde_json::Value, secrets: &[String]) {
match value {
serde_json::Value::String(text) => *text = redact_model_bound_text(text, secrets),
serde_json::Value::Array(values) => {
for value in values {
mask_exact_values(value, secrets);
}
}
serde_json::Value::Object(values) => {
for value in values.values_mut() {
mask_exact_values(value, secrets);
}
}
_ => {}
}
}
// Preserve sensitive-key masking and the existing recursion-depth bound.
let mut redacted = codewhale_config::persistence::redact_json_model_bound_secrets(value);
mask_exact_values(&mut redacted, secrets);
redacted
}
// === Helpers ===
/// How much of a provider's HTTP error body may be shown to the user.
///
/// The catalog probe is the one request that contacts a `base_url` the user
/// typed during setup, and its URL carries an opaque pagination cursor, so a
/// provider — or anything answering at that URL — can echo a key, a custom
/// header value or the cursor back inside an error body. #3385 answered that
/// by discarding the body entirely, and the cost of the blunt version was
/// #6173: Gemini's real reason ("User location is not supported for the API
/// use") reached the user as `Invalid request (400): ` with nothing after the
/// colon, so a geo-block, a bad key and a wrong endpoint were indistinguishable.
pub(super) enum ErrorBodyDisclosure {
/// The endpoint is already established. Surface the provider's message.
Full,
/// Untrusted endpoint. Surface only what survives redaction — and nothing
/// at all if a known secret is still in the result.
Guarded { request_secrets: Vec<String> },
}
/// Read/overall timeout for the shared client's non-streaming requests
/// (`/models` listing, catalog refresh, health probes). Streaming requests
/// keep their own idle-timeout envelope; without this, a provider that accepts
/// the connection and never answers hangs model-list, catalog refresh, and
/// health checks forever (ops R4).
pub(super) const NON_STREAMING_HTTP_TIMEOUT: Duration = Duration::from_secs(30);
pub const PROVIDER_CATALOG_MAX_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
pub const PROVIDER_CATALOG_MAX_ROWS: usize = 10_000;
#[derive(Clone, Copy)]
struct ModelsFetchLimits {
bytes: usize,
rows: usize,
pages: usize,
cursor_bytes: usize,
timeout: Duration,
}
const MODELS_FETCH_LIMITS: ModelsFetchLimits = ModelsFetchLimits {
bytes: PROVIDER_CATALOG_MAX_RESPONSE_BYTES,
rows: PROVIDER_CATALOG_MAX_ROWS,
pages: 1_000,
cursor_bytes: 4_096,
timeout: NON_STREAMING_HTTP_TIMEOUT,
};
#[derive(Clone, Copy)]
enum ModelsRequestMode {
Interactive,
Refresh,
}
#[derive(Debug)]
enum ModelsFetchError {
Catalog(CatalogRefreshError),
Interactive(anyhow::Error),
}
impl ModelsFetchError {
fn into_interactive(self) -> anyhow::Error {
match self {
Self::Catalog(reason) => anyhow::anyhow!("Failed to list models: {reason:?}"),
Self::Interactive(error) => error,
}
}
fn into_catalog(self) -> CatalogRefreshError {
match self {
Self::Catalog(reason) => reason,
Self::Interactive(_) => CatalogRefreshError::Network,
}
}
}
impl From<CatalogRefreshError> for ModelsFetchError {
fn from(reason: CatalogRefreshError) -> Self {
Self::Catalog(reason)
}
}
// Keep row bytes intact: a Value round trip would silently accept duplicate
// fields that the existing typed provider parsers reject.
#[derive(Deserialize)]
struct ModelsPage<'a> {
#[serde(borrow)]
data: Option<&'a serde_json::value::RawValue>,
#[serde(borrow)]
models: Option<&'a serde_json::value::RawValue>,
#[serde(default)]
has_more: bool,
last_id: Option<String>,
}
impl<'a> ModelsPage<'a> {
fn rows(&self) -> Result<&'a serde_json::value::RawValue, CatalogRefreshError> {
self.data
.or(self.models)
.ok_or(CatalogRefreshError::InvalidResponse)
}
}
struct BoundedModelsRows(usize);
impl<'de> serde::de::Visitor<'de> for BoundedModelsRows {
type Value = Vec<Box<serde_json::value::RawValue>>;
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str("a bounded model array")
}
fn visit_seq<A: serde::de::SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
let mut rows = Vec::new();
while rows.len() < self.0 {
let Some(row) = seq.next_element()? else {
return Ok(rows);
};
rows.push(row);
}
if seq.next_element::<serde::de::IgnoredAny>()?.is_some() {
return Err(serde::de::Error::custom("model row limit exceeded"));
}
Ok(rows)
}
}
impl<'de> serde::de::DeserializeSeed<'de> for BoundedModelsRows {
type Value = Vec<Box<serde_json::value::RawValue>>;
fn deserialize<D: serde::Deserializer<'de>>(
self,
deserializer: D,
) -> Result<Self::Value, D::Error> {
deserializer.deserialize_seq(self)
}
}
/// One complete traversal, independent of provider row parsing and transport
/// retry policy. Only a verified endpoint contract supplies a cursor query key;