From 82579667656e4e7b47886e2bd08ed9f78233d374 Mon Sep 17 00:00:00 2001 From: masc-pro-builder Date: Sun, 15 Nov 2026 18:25:26 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix(dashboard):=20=EA=B3=84=EC=82=B0?= =?UTF-8?q?=EC=9D=B4=20=EB=81=9D=EB=82=98=EC=A7=80=20=EB=AA=BB=ED=95=9C=20?= =?UTF-8?q?=EB=92=A4=EC=97=90=EB=8A=94=20tools=20=EC=BA=90=EC=8B=9C?= =?UTF-8?q?=EA=B0=80=20warming=20=EC=94=A8=EC=95=97=EC=9D=84=20=EB=8B=A4?= =?UTF-8?q?=EC=8B=9C=20=EB=82=B4=EB=86=93=EC=A7=80=20=EC=95=8A=EC=95=84?= =?UTF-8?q?=EC=9A=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: tools 엔드포인트는 읽을 때마다 warming 씨앗(Seeded)을 심고, 캐시는 계산이 시간을 넘기거나 실패하면 그 씨앗을 stale 값으로 Ready 에 되살렸어요. 배경 재계산 실패 처리, restore_stale_ready, inline 계산 실패 세 곳이 모두 그랬어요. 그래서 계산이 계속 실패하는 동안 엔드포인트는 error 가 아니라 "warming, 도구 0개"로만 답했어요(#29980). - 세 자리를 entry_after_unfinished 하나로 모았어요. 계산이 끝난 방식은 Gave_up | Timed_out | Raised 로, 값의 출처는 payload_origin 으로 갈라요. - 계산된 값(Computed)과 이미 씨앗 자리를 차지한 Timeout 은 지금처럼 되살려요. - 씨앗은 Gave_up(취소)일 때만 그대로 둬요. Timed_out 이면 timeout 봉투 (origin Timeout)가 씨앗 자리를 차지하고, 계산이 성공할 때까지 stale 값처럼 나가요. Raised 면 씨앗은 grace 가 끝난 채로 자리만 지켜서 다시 심기지 않고, 다음 읽기가 inline 으로 계산해 실패를 그 읽기에 올려요. - timeout 봉투에 창 길이가 필요해서 ?wait_timeout_sec 와 window 를 compute_window = Unbounded | Bounded { timeout_sec; within } 하나로 합쳤어요. 둘은 늘 같은 값이어야 했어요. - 호출자 취소(release_on_cancel)는 손대지 않았어요. Co-Authored-By: Claude Opus 5.5 --- changelog.d/PENDING.md | 10 +++ lib/dashboard/dashboard_cache.ml | 143 ++++++++++++++++++++---------- lib/dashboard/dashboard_cache.mli | 16 +++- test/test_dashboard_cache.ml | 94 ++++++++++++++++++++ 4 files changed, 212 insertions(+), 51 deletions(-) create mode 100644 changelog.d/PENDING.md diff --git a/changelog.d/PENDING.md b/changelog.d/PENDING.md new file mode 100644 index 00000000000..917c21198c9 --- /dev/null +++ b/changelog.d/PENDING.md @@ -0,0 +1,10 @@ +### Fixed + +- The dashboard Tools endpoint reports a failed tools computation instead of + answering `warming` with zero tools indefinitely. The cache put the warming + seed back after every timed-out or failed computation, and the endpoint + plants that seed on each read, so a computation that kept failing looked like + a server that was still starting. A timeout now takes the seed's place and is + served as the endpoint's error until a computation succeeds, and a raised + failure reaches the next reader. A computed stale value is still served + after a failure, as before (#29980, #PENDING). diff --git a/lib/dashboard/dashboard_cache.ml b/lib/dashboard/dashboard_cache.ml index 623cb5fa5bc..ba4fa001e41 100644 --- a/lib/dashboard/dashboard_cache.ml +++ b/lib/dashboard/dashboard_cache.ml @@ -191,10 +191,17 @@ type slot = claim : claim option; } -(* The caller's compute window, applied to whoever runs the compute. *) -type compute_window = { within : 'a. (unit -> 'a) -> 'a } - -let unbounded_window = { within = (fun f -> f ()) } +(* The caller's compute window, applied to whoever runs the compute. A bounded + window also bounds a waiter's wait, and its length is the one a timeout + envelope reports. *) +type compute_window = + | Unbounded + | Bounded of { timeout_sec : float; within : 'a. (unit -> 'a) -> 'a } + +let within_window window f = + match window with + | Unbounded -> f () + | Bounded { within; _ } -> within f let payload_of_entry entry = entry.payload @@ -343,12 +350,6 @@ let is_compute_timeout exn = | Compute_timeout _ -> true | _ -> false -let should_restore_stale_after_failure exn = - match exn with - | Compute_timeout _ -> true - | Eio.Cancel.Cancelled _ -> true - | _ -> false - (* The one shape a timed-out compute takes on the wire. Every timeout path, in this module and outside it, goes through @@ -421,6 +422,54 @@ let timeout_kind_of_role = function | Compute_owner -> Timeout_owner | Compute_waiter -> Timeout_waiter +(* How a compute that held a stale value ended without publishing one. *) +type unfinished_compute = + | Gave_up (* cancelled, or its refresh never started: nothing was learned *) + | Timed_out of { role : compute_timeout_role; timeout_sec : float } + (* the caller's window closed first *) + | Raised (* the compute itself failed *) + +let unfinished_of_exn window exn = + match exn, window with + | Compute_timeout { role; _ }, Bounded { timeout_sec; _ } -> + Timed_out { role; timeout_sec } + (* No window of this cache closed: the compute raised the timeout itself. *) + | Compute_timeout _, Unbounded -> Raised + | Eio.Cancel.Cancelled _, (Bounded _ | Unbounded) -> Gave_up + | _, (Bounded _ | Unbounded) -> Raised + +let origin_of_cached_value = function + | Prepared payload -> payload.origin + | Json_value value -> value.origin + +(* The entry a [Computing] slot goes back to when its compute ends without a + value, from the stale entry the slot held. Every such restore comes here. + + A computed value, or a timeout that already took a seed's place, stays the + fallback for the window the caller passes. + + A seed is no value: it says the key is warming, which is true only until a + compute ends. Restored after that, it answers "warming" for as long as every + compute keeps failing (#29980). + - [Gave_up]: nothing was learned, so the key is still warming. + - [Timed_out]: the timeout envelope takes the seed's place and is served + like any stale value until a compute succeeds. + - [Raised]: the seed stays only so that [seed_stale_if_missing] does not + plant it again. It is past its grace, so the next read computes inline + and the failure reaches that reader, as for a key with no stale value. *) +let entry_after_unfinished ~key ~ts ~expires_at ~stale_until unfinished stale = + match origin_of_cached_value stale.payload, unfinished with + | (Computed | Timeout), (Gave_up | Timed_out _ | Raised) + | Seeded, Gave_up -> { stale with expires_at; stale_until } + | Seeded, Timed_out { role; timeout_sec } -> + let envelope = + timeout_envelope_json ~kind:(timeout_kind_of_role role) ~key ~timeout_sec + in + { payload = Prepared (payload_of_json ~origin:Timeout envelope); + expires_at; + stale_until } + | Seeded, Raised -> { stale with expires_at = ts; stale_until = ts } + (** Maximum seconds a waiter will poll for a [Computing] slot before evicting it and recomputing. @@ -520,7 +569,7 @@ let inc_cache_miss () = A pool job that would wait on a compute whose owner's job has not started claims that compute and runs its own instead of holding its worker idle; the owner's job then returns without computing (RFC-0204 3.4 L4b). *) -let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl produce = +let get_or_compute_eio ?(window = Unbounded) key ~ttl produce = let stale_grace = ttl *. stale_factor in let started = now () in (* A runner that was cancelled or whose window closed gave up on the @@ -532,7 +581,7 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p in (* A pool job of the work's kind unless the caller is already inside one. *) let compute () = - window.within (fun () -> Pool_work.submit produce) + within_window window (fun () -> Pool_work.submit produce) in let clear_claimed ~token = atomic_update table (fun map -> @@ -581,7 +630,7 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p | exception _ -> () (* settled as [Failed]; the owner raises it below *)) in match - window.within (fun () -> + within_window window (fun () -> Eio.Fiber.first (fun () -> Pool_work.submit job; @@ -629,9 +678,9 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p in let elapsed = now () -. started_at in let timed_out_waiter = - match wait_timeout_sec with - | Some timeout_sec -> waited >= timeout_sec - | None -> false + match window with + | Bounded { timeout_sec; _ } -> waited >= timeout_sec + | Unbounded -> false in let watchdog_ceiling = max_wait_sec () in if timed_out_waiter then @@ -725,11 +774,11 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p let expires_at = if Cancel_safe.is_internal_race_cancel exn then ts else ts +. backoff_grace in - let refreshed = { - stale_entry with - expires_at; - stale_until = ts +. backoff_grace; - } in + let refreshed = + entry_after_unfinished ~key ~ts ~expires_at + ~stale_until:(ts +. backoff_grace) + (unfinished_of_exn window exn) stale_entry + in ((), SMap.add key (Ready refreshed) map) | Some (Ready _) | None | Some (Computing _) -> ((), map) ) @@ -739,11 +788,10 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p | Some (Computing { token = c; _ }) when c = token -> let ts = now () in let backoff_grace = stale_grace *. bg_revalidate_backoff_factor in - let restored = { - stale_entry with - expires_at = ts; - stale_until = ts +. backoff_grace; - } in + let restored = + entry_after_unfinished ~key ~ts ~expires_at:ts + ~stale_until:(ts +. backoff_grace) Gave_up stale_entry + in ((), SMap.add key (Ready restored) map) | Some (Ready _) | None | Some (Computing _) -> ((), map) ) @@ -872,28 +920,28 @@ let get_or_compute_eio ?wait_timeout_sec ?(window = unbounded_window) key ~ttl p ); new_entry | Some (Error exn) -> - let fallback_val = ref None in + let unfinished = unfinished_of_exn window exn in if not (Cancel_safe.is_internal_race_cancel exn || is_compute_timeout exn) then Log.Dashboard.error "cache revalidation failed: %s" (Printexc.to_string exn); - atomic_update table (fun map -> - match SMap.find_opt key map with - | Some (Computing { token = c; stale = Some stale_value; _ }) when c = token -> - let backoff_grace = stale_grace *. bg_revalidate_backoff_factor in - let refreshed = { - stale_value with - expires_at = ts; - stale_until = ts +. backoff_grace; - } in - fallback_val := Some refreshed; - ((), SMap.add key (Ready refreshed) map) - | Some (Computing { token = c; stale = None; _ }) when c = token -> - ((), SMap.remove key map) - | Some (Ready _) | None | Some (Computing _) -> ((), map) - ); - (match !fallback_val with - | Some entry when should_restore_stale_after_failure exn -> - entry - | _ -> raise exn) + let fallback = + atomic_update table (fun map -> + match SMap.find_opt key map with + | Some (Computing { token = c; stale = Some stale_value; _ }) when c = token -> + let backoff_grace = stale_grace *. bg_revalidate_backoff_factor in + let refreshed = + entry_after_unfinished ~key ~ts ~expires_at:ts + ~stale_until:(ts +. backoff_grace) unfinished stale_value + in + (Some refreshed, SMap.add key (Ready refreshed) map) + | Some (Computing { token = c; stale = None; _ }) when c = token -> + (None, SMap.remove key map) + | Some (Ready _) | None | Some (Computing _) -> (None, map)) + in + (* A timeout or a cancellation answers with the fallback; a compute + that raised reaches its caller. *) + (match fallback, unfinished with + | Some entry, (Timed_out _ | Gave_up) -> entry + | Some _, Raised | None, (Timed_out _ | Gave_up | Raised) -> raise exn) | None -> let fallback_val = ref None in let fallback_payload = Prepared (payload_of_json ~origin:Timeout @@ -1070,8 +1118,7 @@ let get_or_compute_value_with_timeout ~preparation try let entry = if Eio_guard.is_ready () then - get_or_compute_eio ~wait_timeout_sec:timeout_sec - ~window:{ within } key ~ttl produce + get_or_compute_eio ~window:(Bounded { timeout_sec; within }) key ~ttl produce else get_or_compute_simple key ~ttl (fun () -> with_timeout compute) in diff --git a/lib/dashboard/dashboard_cache.mli b/lib/dashboard/dashboard_cache.mli index 177e9b012ac..67455508a00 100644 --- a/lib/dashboard/dashboard_cache.mli +++ b/lib/dashboard/dashboard_cache.mli @@ -130,7 +130,8 @@ val get_or_compute_with_timeout : Yojson.Safe.t Pool_work.t -> Yojson.Safe.t (** Like [get_or_compute] but runs the work under an Eio timeout. On timeout in the stale-while-revalidate path, the stale value is - preserved (not overwritten by error JSON). On timeout with no stale + preserved (not overwritten by error JSON), unless that value is a seed + ({!seed_stale_if_missing}). On timeout with no stale data, returns a timeout-error JSON without caching it. Repeated no-stale timeouts for the same key open a short fail-fast circuit; fresh or stale cache entries are still served normally. *) @@ -141,7 +142,8 @@ val get_or_compute_payload_with_timeout : Yojson.Safe.t Pool_work.t -> cached_payload (** Same timeout and stale-value semantics as {!get_or_compute_with_timeout}, retaining the memoized bytes and ETag of the returned JSON. Timeout envelopes - are returned uncached. Preparation completes inside the timed worker compute + are returned uncached, except one that takes a seed's place + ({!seed_stale_if_missing}). Preparation completes inside the timed worker compute before the cache entry is published. Prepared hits and stale responses do not wait for executor capacity. Preparing an existing JSON-only value also obeys [timeout_sec]; cancellation leaves the AST usable and permits a later @@ -168,7 +170,15 @@ val seed_stale_if_missing : background when a switch is available. Existing entries are left intact. Seeds are intended for small warming responses: their bytes and ETag are prepared synchronously before publication, without using the executor pool. - An existing slot skips preparation; a slot installed concurrently wins. *) + An existing slot skips preparation; a slot installed concurrently wins. + + A seed answers only until a compute for [key] ends. A compute that + succeeds replaces it. A cancelled one leaves it in place. After a compute + times out, the timeout envelope (origin [Timeout]) takes its place and is + served like a stale value until a compute succeeds. After a compute + raises, the seed is no longer served: the next read computes inline and + the failure reaches that reader. In both cases the key stays occupied, so + seeding again does not bring the seed back. *) val seed_stale_from_if_missing : string -> from:string -> stale_for:float -> unit (** [seed_stale_from_if_missing key ~from ~stale_for] puts [from]'s current diff --git a/test/test_dashboard_cache.ml b/test/test_dashboard_cache.ml index 68d982d4f8e..3407bb383f0 100644 --- a/test/test_dashboard_cache.ml +++ b/test/test_dashboard_cache.ml @@ -1250,6 +1250,94 @@ let test_waiter_timeout_returns_error_not_cached ~clock () = in check_json "owner result survives waiter timeout" (`String "owner_done") final +(* -- 14. A seed does not outlive the compute it was waiting for ------------- *) + +(* #29980. A seed says the key is warming, which is true only until a compute + ends. The Tools page seeds its key on every read, so a seed that comes back + after a failed compute answers "warming, no tools" for as long as every + compute keeps failing. These reads go through the payload API because the + origin is what the Tools page turns into pending, ready or error. *) + +let await_no_computing_slot ~clock = + Eio.Time.with_timeout_exn clock 2. (fun () -> + let rec loop () = + if Yojson.Safe.Util.(Dashboard_cache.stats () |> member "computing" |> to_int) > 0 + then (Eio.Time.sleep clock 0.01; loop ()) + in + loop ()) + +let check_origin msg expected (payload : Dashboard_cache.cached_payload) = + let name = function + | Dashboard_cache.Seeded -> "seeded" + | Computed -> "computed" + | Timeout -> "timeout" + in + Alcotest.(check string) msg (name expected) (name payload.origin) + +let test_seed_gives_way_to_a_background_timeout ~clock () = + Dashboard_cache.invalidate_all (); + let key = "seed-background-timeout" in + Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming"); + let read () = + Dashboard_cache.get_or_compute_payload_with_timeout key ~ttl:60. ~clock + ~timeout_sec:0.05 (Pool_work.cpu ~name:__FUNCTION__ (fun () -> + Eio.Time.sleep clock 1.0; + `String "never_reached")) + in + check_origin "the first read answers with the seed" Dashboard_cache.Seeded (read ()); + await_no_computing_slot ~clock; + let after = read () in + check_origin "the read after the timed-out refresh reports the timeout" + Dashboard_cache.Timeout after; + Alcotest.(check string) "with the refresh's own timeout" "owner" (timeout_kind after.json); + Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming"); + check_origin "a seed planted after the timeout does not replace it" + Dashboard_cache.Timeout (read ()) + +let test_expired_seed_reports_an_inline_timeout ~clock () = + Dashboard_cache.invalidate_all (); + let key = "seed-inline-timeout" in + (* Past its grace at once, so the read computes inline with the seed as the + slot's only fallback. *) + Dashboard_cache.seed_stale_if_missing key ~stale_for:0. (`String "warming"); + let answer = + Dashboard_cache.get_or_compute_payload_with_timeout key ~ttl:60. ~clock + ~timeout_sec:0.05 (Pool_work.cpu ~name:__FUNCTION__ (fun () -> + Eio.Time.sleep clock 1.0; + `String "never_reached")) + in + check_origin "the read whose compute timed out reports the timeout" + Dashboard_cache.Timeout answer; + check_json "and the slot keeps the timeout, not the seed" + answer.json (Option.get (Dashboard_cache.peek key)) + +let test_seed_gives_way_to_a_background_failure ~clock () = + Dashboard_cache.invalidate_all (); + let key = "seed-background-failure" in + Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming"); + let read work = + Dashboard_cache.get_or_compute_payload_with_timeout key ~ttl:60. ~clock + ~timeout_sec:2. (Pool_work.cpu ~name:__FUNCTION__ work) + in + let unreadable () = failwith "tools unreadable" in + check_origin "the first read answers with the seed" Dashboard_cache.Seeded + (read unreadable); + await_no_computing_slot ~clock; + (match read unreadable with + | payload -> + Alcotest.failf "the failed refresh was answered as %s" payload.raw_json + | exception Failure message -> + Alcotest.(check string) "the next reader gets the failure" "tools unreadable" message); + Dashboard_cache.seed_stale_if_missing key ~stale_for:60. (`String "warming"); + (match read unreadable with + | payload -> + Alcotest.failf "a seed planted after the failure answered as %s" payload.raw_json + | exception Failure _ -> ()); + let recovered = read (fun () -> `String "tools") in + check_origin "a compute that succeeds answers with its value" + Dashboard_cache.Computed recovered; + check_json "the computed tools" (`String "tools") recovered.json + let with_temp_dir prefix f = let dir = Filename.temp_file prefix "" in Sys.remove dir; @@ -2229,6 +2317,12 @@ let () = (test_repeated_no_stale_timeout_opens_circuit ~clock); test_case "waiter timeout returns error, not cached" `Quick (test_waiter_timeout_returns_error_not_cached ~clock); + test_case "a seed gives way to a background timeout" `Quick + (test_seed_gives_way_to_a_background_timeout ~clock); + test_case "an expired seed reports an inline timeout" `Quick + (test_expired_seed_reports_an_inline_timeout ~clock); + test_case "a seed gives way to a background failure" `Quick + (test_seed_gives_way_to_a_background_failure ~clock); test_case "timeout envelope producer and recognizer agree" `Quick (test_timeout_envelope_recognizer ~clock); ] ); From f68e9f461995a4461511109df642583ffbb3c9de Mon Sep 17 00:00:00 2001 From: masc-pro-builder Date: Sun, 15 Nov 2026 18:28:34 +0900 Subject: [PATCH 2/2] =?UTF-8?q?changelog:=20#52269=20=EC=A1=B0=EA=B0=81?= =?UTF-8?q?=EC=9D=98=20=EC=9D=B4=EB=A6=84=EC=9D=84=20PR=20=EB=B2=88?= =?UTF-8?q?=ED=98=B8=EB=A1=9C=20=EB=A7=9E=EC=B6=B0=EC=9A=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- changelog.d/{PENDING.md => 52269.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename changelog.d/{PENDING.md => 52269.md} (91%) diff --git a/changelog.d/PENDING.md b/changelog.d/52269.md similarity index 91% rename from changelog.d/PENDING.md rename to changelog.d/52269.md index 917c21198c9..e2731d99599 100644 --- a/changelog.d/PENDING.md +++ b/changelog.d/52269.md @@ -7,4 +7,4 @@ a server that was still starting. A timeout now takes the seed's place and is served as the endpoint's error until a computation succeeds, and a raised failure reaches the next reader. A computed stale value is still served - after a failure, as before (#29980, #PENDING). + after a failure, as before (#29980, #52269).