Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions changelog.d/52269.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
### Fixed

- The dashboard Tools endpoint reports a failed tools computation instead of
answering `warming` with zero tools indefinitely. The cache put the warming
seed back after every timed-out or failed computation, and the endpoint
plants that seed on each read, so a computation that kept failing looked like
a server that was still starting. A timeout now takes the seed's place and is
served as the endpoint's error until a computation succeeds, and a raised
failure reaches the next reader. A computed stale value is still served
after a failure, as before (#29980, #52269).
143 changes: 95 additions & 48 deletions lib/dashboard/dashboard_cache.ml
Original file line number Diff line number Diff line change
Expand Up @@ -191,10 +191,17 @@ type slot =
claim : claim option;
}

(* The caller's compute window, applied to whoever runs the compute. *)
type compute_window = { within : 'a. (unit -> 'a) -> 'a }

let unbounded_window = { within = (fun f -> f ()) }
(* The caller's compute window, applied to whoever runs the compute. A bounded
window also bounds a waiter's wait, and its length is the one a timeout
envelope reports. *)
type compute_window =
| Unbounded
| Bounded of { timeout_sec : float; within : 'a. (unit -> 'a) -> 'a }

let within_window window f =
match window with
| Unbounded -> f ()
| Bounded { within; _ } -> within f

let payload_of_entry entry = entry.payload

Expand Down Expand Up @@ -343,12 +350,6 @@ let is_compute_timeout exn =
| Compute_timeout _ -> true
| _ -> false

let should_restore_stale_after_failure exn =
match exn with
| Compute_timeout _ -> true
| Eio.Cancel.Cancelled _ -> true
| _ -> false

(* The one shape a timed-out compute takes on the wire.

Every timeout path, in this module and outside it, goes through
Expand Down Expand Up @@ -421,6 +422,54 @@ let timeout_kind_of_role = function
| Compute_owner -> Timeout_owner
| Compute_waiter -> Timeout_waiter

(* How a compute that held a stale value ended without publishing one. *)
type unfinished_compute =
| Gave_up (* cancelled, or its refresh never started: nothing was learned *)
| Timed_out of { role : compute_timeout_role; timeout_sec : float }
(* the caller's window closed first *)
| Raised (* the compute itself failed *)

let unfinished_of_exn window exn =
match exn, window with
| Compute_timeout { role; _ }, Bounded { timeout_sec; _ } ->
Timed_out { role; timeout_sec }
(* No window of this cache closed: the compute raised the timeout itself. *)
| Compute_timeout _, Unbounded -> Raised
| Eio.Cancel.Cancelled _, (Bounded _ | Unbounded) -> Gave_up
| _, (Bounded _ | Unbounded) -> Raised

let origin_of_cached_value = function
| Prepared payload -> payload.origin
| Json_value value -> value.origin

(* The entry a [Computing] slot goes back to when its compute ends without a
value, from the stale entry the slot held. Every such restore comes here.

A computed value, or a timeout that already took a seed's place, stays the
fallback for the window the caller passes.

A seed is no value: it says the key is warming, which is true only until a
compute ends. Restored after that, it answers "warming" for as long as every
compute keeps failing (#29980).
- [Gave_up]: nothing was learned, so the key is still warming.
- [Timed_out]: the timeout envelope takes the seed's place and is served
like any stale value until a compute succeeds.
- [Raised]: the seed stays only so that [seed_stale_if_missing] does not
plant it again. It is past its grace, so the next read computes inline
and the failure reaches that reader, as for a key with no stale value. *)
let entry_after_unfinished ~key ~ts ~expires_at ~stale_until unfinished stale =
match origin_of_cached_value stale.payload, unfinished with
| (Computed | Timeout), (Gave_up | Timed_out _ | Raised)
| Seeded, Gave_up -> { stale with expires_at; stale_until }
| Seeded, Timed_out { role; timeout_sec } ->
let envelope =
timeout_envelope_json ~kind:(timeout_kind_of_role role) ~key ~timeout_sec
in
{ payload = Prepared (payload_of_json ~origin:Timeout envelope);
expires_at;
stale_until }
| Seeded, Raised -> { stale with expires_at = ts; stale_until = ts }

(** Maximum seconds a waiter will poll for a [Computing] slot before evicting
it and recomputing.

Expand Down Expand Up @@ -520,7 +569,7 @@ let inc_cache_miss () =
A pool job that would wait on a compute whose owner's job has not started
claims that compute and runs its own instead of holding its worker idle;
the owner's job then returns without computing (RFC-0204 3.4 L4b). *)
let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl produce =
let get_or_compute_eio ?(window = Unbounded) key ~ttl produce =
let stale_grace = ttl *. stale_factor in
let started = now () in
(* A runner that was cancelled or whose window closed gave up on the
Expand All @@ -532,7 +581,7 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p
in
(* A pool job of the work's kind unless the caller is already inside one. *)
let compute () =
window.within (fun () -> Pool_work.submit produce)
within_window window (fun () -> Pool_work.submit produce)
in
let clear_claimed ~token =
atomic_update table (fun map ->
Expand Down Expand Up @@ -581,7 +630,7 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p
| exception _ -> () (* settled as [Failed]; the owner raises it below *))
in
match
window.within (fun () ->
within_window window (fun () ->
Eio.Fiber.first
(fun () ->
Pool_work.submit job;
Expand Down Expand Up @@ -629,9 +678,9 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p
in
let elapsed = now () -. started_at in
let timed_out_waiter =
match wait_timeout_sec with
| Some timeout_sec -> waited >= timeout_sec
| None -> false
match window with
| Bounded { timeout_sec; _ } -> waited >= timeout_sec
| Unbounded -> false
in
let watchdog_ceiling = max_wait_sec () in
if timed_out_waiter then
Expand Down Expand Up @@ -725,11 +774,11 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p
let expires_at =
if Cancel_safe.is_internal_race_cancel exn then ts else ts +. backoff_grace
in
let refreshed = {
stale_entry with
expires_at;
stale_until = ts +. backoff_grace;
} in
let refreshed =
entry_after_unfinished ~key ~ts ~expires_at
~stale_until:(ts +. backoff_grace)
(unfinished_of_exn window exn) stale_entry
in
((), SMap.add key (Ready refreshed) map)
| Some (Ready _) | None | Some (Computing _) -> ((), map)
)
Expand All @@ -739,11 +788,10 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p
| Some (Computing { token = c; _ }) when c = token ->
let ts = now () in
let backoff_grace = stale_grace *. bg_revalidate_backoff_factor in
let restored = {
stale_entry with
expires_at = ts;
stale_until = ts +. backoff_grace;
} in
let restored =
entry_after_unfinished ~key ~ts ~expires_at:ts
~stale_until:(ts +. backoff_grace) Gave_up stale_entry
in
((), SMap.add key (Ready restored) map)
| Some (Ready _) | None | Some (Computing _) -> ((), map)
)
Expand Down Expand Up @@ -872,28 +920,28 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p
);
new_entry
| Some (Error exn) ->
let fallback_val = ref None in
let unfinished = unfinished_of_exn window exn in
if not (Cancel_safe.is_internal_race_cancel exn || is_compute_timeout exn) then
Log.Dashboard.error "cache revalidation failed: %s" (Printexc.to_string exn);
atomic_update table (fun map ->
match SMap.find_opt key map with
| Some (Computing { token = c; stale = Some stale_value; _ }) when c = token ->
let backoff_grace = stale_grace *. bg_revalidate_backoff_factor in
let refreshed = {
stale_value with
expires_at = ts;
stale_until = ts +. backoff_grace;
} in
fallback_val := Some refreshed;
((), SMap.add key (Ready refreshed) map)
| Some (Computing { token = c; stale = None; _ }) when c = token ->
((), SMap.remove key map)
| Some (Ready _) | None | Some (Computing _) -> ((), map)
);
(match !fallback_val with
| Some entry when should_restore_stale_after_failure exn ->
entry
| _ -> raise exn)
let fallback =
atomic_update table (fun map ->
match SMap.find_opt key map with
| Some (Computing { token = c; stale = Some stale_value; _ }) when c = token ->
let backoff_grace = stale_grace *. bg_revalidate_backoff_factor in
let refreshed =
entry_after_unfinished ~key ~ts ~expires_at:ts
~stale_until:(ts +. backoff_grace) unfinished stale_value
in
(Some refreshed, SMap.add key (Ready refreshed) map)
| Some (Computing { token = c; stale = None; _ }) when c = token ->
(None, SMap.remove key map)
| Some (Ready _) | None | Some (Computing _) -> (None, map))
in
(* A timeout or a cancellation answers with the fallback; a compute
that raised reaches its caller. *)
(match fallback, unfinished with
| Some entry, (Timed_out _ | Gave_up) -> entry
| Some _, Raised | None, (Timed_out _ | Gave_up | Raised) -> raise exn)
| None ->
let fallback_val = ref None in
let fallback_payload = Prepared (payload_of_json ~origin:Timeout
Expand Down Expand Up @@ -1070,8 +1118,7 @@ let get_or_compute_value_with_timeout ~preparation
try
let entry =
if Eio_guard.is_ready () then
get_or_compute_eio ~wait_timeout_sec:timeout_sec
~window:{ within } key ~ttl produce
get_or_compute_eio ~window:(Bounded { timeout_sec; within }) key ~ttl produce
else
get_or_compute_simple key ~ttl (fun () -> with_timeout compute)
in
Expand Down
16 changes: 13 additions & 3 deletions lib/dashboard/dashboard_cache.mli
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ val get_or_compute_with_timeout :
Yojson.Safe.t Pool_work.t -> Yojson.Safe.t
(** Like [get_or_compute] but runs the work under an Eio timeout.
On timeout in the stale-while-revalidate path, the stale value is
preserved (not overwritten by error JSON). On timeout with no stale
preserved (not overwritten by error JSON), unless that value is a seed
({!seed_stale_if_missing}). On timeout with no stale
data, returns a timeout-error JSON without caching it. Repeated no-stale
timeouts for the same key open a short fail-fast circuit; fresh or stale
cache entries are still served normally. *)
Expand All @@ -141,7 +142,8 @@ val get_or_compute_payload_with_timeout :
Yojson.Safe.t Pool_work.t -> cached_payload
(** Same timeout and stale-value semantics as {!get_or_compute_with_timeout},
retaining the memoized bytes and ETag of the returned JSON. Timeout envelopes
are returned uncached. Preparation completes inside the timed worker compute
are returned uncached, except one that takes a seed's place
({!seed_stale_if_missing}). Preparation completes inside the timed worker compute
before the cache entry is published. Prepared hits and stale responses do
not wait for executor capacity. Preparing an existing JSON-only value also
obeys [timeout_sec]; cancellation leaves the AST usable and permits a later
Expand All @@ -168,7 +170,15 @@ val seed_stale_if_missing :
background when a switch is available. Existing entries are left intact.
Seeds are intended for small warming responses: their bytes and ETag are
prepared synchronously before publication, without using the executor pool.
An existing slot skips preparation; a slot installed concurrently wins. *)
An existing slot skips preparation; a slot installed concurrently wins.

A seed answers only until a compute for [key] ends. A compute that
succeeds replaces it. A cancelled one leaves it in place. After a compute
times out, the timeout envelope (origin [Timeout]) takes its place and is
served like a stale value until a compute succeeds. After a compute
raises, the seed is no longer served: the next read computes inline and
the failure reaches that reader. In both cases the key stays occupied, so
seeding again does not bring the seed back. *)

val seed_stale_from_if_missing : string -> from:string -> stale_for:float -> unit
(** [seed_stale_from_if_missing key ~from ~stale_for] puts [from]'s current
Expand Down
94 changes: 94 additions & 0 deletions test/test_dashboard_cache.ml
Original file line number Diff line number Diff line change
Expand Up @@ -1250,6 +1250,94 @@ let test_waiter_timeout_returns_error_not_cached ~clock () =
in
check_json "owner result survives waiter timeout" (`String "owner_done") final

(* -- 14. A seed does not outlive the compute it was waiting for ------------- *)

(* #29980. A seed says the key is warming, which is true only until a compute
ends. The Tools page seeds its key on every read, so a seed that comes back
after a failed compute answers "warming, no tools" for as long as every
compute keeps failing. These reads go through the payload API because the
origin is what the Tools page turns into pending, ready or error. *)

let await_no_computing_slot ~clock =
Eio.Time.with_timeout_exn clock 2. (fun () ->
let rec loop () =
if Yojson.Safe.Util.(Dashboard_cache.stats () |> member "computing" |> to_int) > 0
then (Eio.Time.sleep clock 0.01; loop ())
in
loop ())

let check_origin msg expected (payload : Dashboard_cache.cached_payload) =
let name = function
| Dashboard_cache.Seeded -> "seeded"
| Computed -> "computed"
| Timeout -> "timeout"
in
Alcotest.(check string) msg (name expected) (name payload.origin)

let test_seed_gives_way_to_a_background_timeout ~clock () =
Dashboard_cache.invalidate_all ();
let key = "seed-background-timeout" in
Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming");
let read () =
Dashboard_cache.get_or_compute_payload_with_timeout key ~ttl:60. ~clock
~timeout_sec:0.05 (Pool_work.cpu ~name:__FUNCTION__ (fun () ->
Eio.Time.sleep clock 1.0;
`String "never_reached"))
in
check_origin "the first read answers with the seed" Dashboard_cache.Seeded (read ());
await_no_computing_slot ~clock;
let after = read () in
check_origin "the read after the timed-out refresh reports the timeout"
Dashboard_cache.Timeout after;
Alcotest.(check string) "with the refresh's own timeout" "owner" (timeout_kind after.json);
Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming");
check_origin "a seed planted after the timeout does not replace it"
Dashboard_cache.Timeout (read ())

let test_expired_seed_reports_an_inline_timeout ~clock () =
Dashboard_cache.invalidate_all ();
let key = "seed-inline-timeout" in
(* Past its grace at once, so the read computes inline with the seed as the
slot's only fallback. *)
Dashboard_cache.seed_stale_if_missing key ~stale_for:0. (`String "warming");
let answer =
Dashboard_cache.get_or_compute_payload_with_timeout key ~ttl:60. ~clock
~timeout_sec:0.05 (Pool_work.cpu ~name:__FUNCTION__ (fun () ->
Eio.Time.sleep clock 1.0;
`String "never_reached"))
in
check_origin "the read whose compute timed out reports the timeout"
Dashboard_cache.Timeout answer;
check_json "and the slot keeps the timeout, not the seed"
answer.json (Option.get (Dashboard_cache.peek key))

let test_seed_gives_way_to_a_background_failure ~clock () =
Dashboard_cache.invalidate_all ();
let key = "seed-background-failure" in
Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming");
let read work =
Dashboard_cache.get_or_compute_payload_with_timeout key ~ttl:60. ~clock
~timeout_sec:2. (Pool_work.cpu ~name:__FUNCTION__ work)
in
let unreadable () = failwith "tools unreadable" in
check_origin "the first read answers with the seed" Dashboard_cache.Seeded
(read unreadable);
await_no_computing_slot ~clock;
(match read unreadable with
| payload ->
Alcotest.failf "the failed refresh was answered as %s" payload.raw_json
| exception Failure message ->
Alcotest.(check string) "the next reader gets the failure" "tools unreadable" message);
Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming");
(match read unreadable with
| payload ->
Alcotest.failf "a seed planted after the failure answered as %s" payload.raw_json
| exception Failure _ -> ());
let recovered = read (fun () -> `String "tools") in
check_origin "a compute that succeeds answers with its value"
Dashboard_cache.Computed recovered;
check_json "the computed tools" (`String "tools") recovered.json

let with_temp_dir prefix f =
let dir = Filename.temp_file prefix "" in
Sys.remove dir;
Expand Down Expand Up @@ -2229,6 +2317,12 @@ let () =
(test_repeated_no_stale_timeout_opens_circuit ~clock);
test_case "waiter timeout returns error, not cached" `Quick
(test_waiter_timeout_returns_error_not_cached ~clock);
test_case "a seed gives way to a background timeout" `Quick
(test_seed_gives_way_to_a_background_timeout ~clock);
test_case "an expired seed reports an inline timeout" `Quick
(test_expired_seed_reports_an_inline_timeout ~clock);
test_case "a seed gives way to a background failure" `Quick
(test_seed_gives_way_to_a_background_failure ~clock);
test_case "timeout envelope producer and recognizer agree" `Quick
(test_timeout_envelope_recognizer ~clock);
] );
Expand Down
Loading