From 193a173c425035820f7778a101d3e19de9dfa9b6 Mon Sep 17 00:00:00 2001 From: zakrepashe Date: Thu, 23 Jul 2026 22:17:59 +0300 Subject: [PATCH 1/2] fix(cookbook): stop local Windows download orphans that deadlock HF retries Client-side tree-kill alone could leave hf/python children holding cache locks after Stop. Centralize stop in /api/cookbook/stop-session with verified local Windows kills, orphan scans, stop markers, and a same-repo live-download guard so retries reattach or refuse instead of deadlocking. Co-authored-by: Cursor --- routes/cookbook_routes.py | 591 ++++++++++++++++++++- src/tools/system.py | 3 + static/js/cookbookRunning.js | 254 ++++++--- tests/test_cookbook_download_restart_js.py | 52 ++ tests/test_cookbook_helpers.py | 8 +- tests/test_cookbook_local_windows_stop.py | 217 ++++++++ 6 files changed, 1041 insertions(+), 84 deletions(-) create mode 100644 tests/test_cookbook_download_restart_js.py create mode 100644 tests/test_cookbook_local_windows_stop.py diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 1d79ba809..34b4234ce 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -47,7 +47,7 @@ logger = logging.getLogger(__name__) from routes.cookbook_helpers import ( - _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token, + _SESSION_ID_RE, _REPO_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token, _validate_local_dir, _validate_gpus, _shell_path, _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT, _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines, @@ -370,6 +370,35 @@ def _append_local_ollama_download_command_lines( lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi') +def _cmdline_references_hf_repo(text: str, repo_id: str) -> bool: + """True when text references an HF download of repo_id (exact id, not a prefix).""" + if not text or not repo_id: + return False + boundary = r"(?![A-Za-z0-9._-])" + escaped = re.escape(repo_id) + patterns = ( + # hf / hf.exe / "…\hf.exe" / '…\hf.exe' download + # Optional closing quote after .exe — Windows often quotes the executable path. + rf"\bhf(?:\.exe)?[\"']?\s+download\s+{escaped}{boundary}", + # hf_download.py script arg — quoted/unquoted, optional python -u prefix + rf"\bhf_download\.py[\"']?\s+{escaped}{boundary}", + # Python huggingface_hub fallback — positional and repo_id= forms. + # Require the matching closing quote plus ',' or ')' so org/model + # never matches a concurrently downloading org/model-large. + rf"\bsnapshot_download\s*\(\s*([\"']){escaped}\1\s*(?:,|\))", + rf"\bsnapshot_download\s*\(\s*repo_id\s*=\s*([\"']){escaped}\1\s*(?:,|\))", + ) + return any(re.search(p, text, re.IGNORECASE) for p in patterns) + + +def _coerce_ssh_port(v: str | None) -> str | None: + """Non-throwing ssh port check; returns normalized port or None.""" + try: + return validate_ssh_port(None if v in (None, "") else str(v)) + except HTTPException: + return None + + def setup_cookbook_routes() -> APIRouter: router = APIRouter(tags=["cookbook"]) _cookbook_state_path = Path(COOKBOOK_STATE_FILE) @@ -1023,6 +1052,473 @@ def _launch_local_detached(session_id: str, bash_lines: list[str]) -> dict: pid_path.write_text(str(proc.pid), encoding="utf-8") return {"pid": proc.pid, "log_path": str(log_path)} + def _windows_download_process_scan_ps() -> str: + return ( + "Get-CimInstance Win32_Process -Filter " + "\"Name='python.exe' OR Name='hf.exe'\" | " + "ForEach-Object { \"$($_.ProcessId)`t$($_.CommandLine)\" }" + ) + + def _parse_windows_download_scan(stdout: str, repo_id: str) -> int | None: + for line in (stdout or "").splitlines(): + pid_s, _, cmdline = line.partition("\t") + if _cmdline_references_hf_repo(cmdline, repo_id): + try: + return int(pid_s.strip()) + except ValueError: + continue + return None + + def _scan_windows_download_processes_checked( + repo_id: str, + ) -> tuple[bool, int | None]: + """Return (scan_succeeded, downloader_pid) for a local Windows repo.""" + try: + probe = subprocess.run( + ["powershell", "-NoProfile", "-Command", _windows_download_process_scan_ps()], + capture_output=True, + text=True, + timeout=15, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except Exception: + return False, None + if probe.returncode != 0: + return False, None + out = probe.stdout or "" + return True, _parse_windows_download_scan(out, repo_id) + + def _scan_windows_download_processes(repo_id: str) -> int | None: + """PID of any live process downloading repo_id (Windows, best-effort). + + Catches downloaders that cookbook session files no longer track — + e.g. a stop issued from a stale browser tab running pre-tree-kill JS + deleted the .pid file but left the hf/python children running. Those + orphans hold the HF cache file locks and deadlock any new download of + the same repo. + """ + _scan_ok, pid = _scan_windows_download_processes_checked(repo_id) + return pid + + def _ssh_powershell_argv(remote: str, ssh_port: str | None, ps: str) -> list[str]: + ssh_args = [ + "ssh", + "-o", "ConnectTimeout=5", + "-o", "StrictHostKeyChecking=no", + ] + safe_port = _coerce_ssh_port(ssh_port) + if safe_port and safe_port != "22": + ssh_args.extend(["-p", safe_port]) + ssh_args.extend([remote, "powershell", "-NoProfile", "-Command", ps]) + return ssh_args + + async def _ssh_powershell(remote: str, ssh_port: str | None, ps: str) -> tuple[int, str]: + """Run inline PowerShell on a remote Windows host via ssh argv (no local shell).""" + try: + proc = await asyncio.create_subprocess_exec( + *_ssh_powershell_argv(remote, ssh_port, ps), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _stderr = await asyncio.wait_for(proc.communicate(), timeout=20) + except Exception: + return 1, "" + return proc.returncode or 0, (stdout or b"").decode("utf-8", errors="replace") + + async def _scan_remote_windows_download_processes( + remote: str, ssh_port: str | None, repo_id: str + ) -> int | None: + """PID of a live remote-Windows downloader for repo_id (best-effort).""" + _rc, out = await _ssh_powershell(remote, ssh_port, _windows_download_process_scan_ps()) + return _parse_windows_download_scan(out, repo_id) + + def _local_ps1_sessions_for_repo(repo_id: str) -> list[str]: + """Session ids whose local _run.ps1 references an HF download of repo_id.""" + try: + runners = sorted(TMUX_LOG_DIR.glob("cookbook-*_run.ps1")) + except OSError: + return [] + sids: list[str] = [] + for path in runners: + sid = path.name.removesuffix("_run.ps1") + if not _SESSION_ID_RE.match(sid): + continue + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + if _cmdline_references_hf_repo(text, repo_id): + sids.append(sid) + return sids + def _server_platform_for_host(remote_host: str | None) -> str: + """Platform string from cookbook server profiles for a remote host.""" + if not remote_host or not _cookbook_state_path.exists(): + return "" + try: + state = json.loads(_cookbook_state_path.read_text(encoding="utf-8")) + except Exception: + return "" + env_state = state.get("env") if isinstance(state, dict) else {} + servers = env_state.get("servers") if isinstance(env_state, dict) else [] + if not isinstance(servers, list): + return "" + for server in servers: + if isinstance(server, dict) and (server.get("host") or "").strip() == remote_host: + return (server.get("platform") or "").strip().lower() + return "" + + def _resolve_windows_platform(remote_host: str | None, platform: str | None) -> str: + plat = (platform or "").strip().lower() + if remote_host: + configured = _server_platform_for_host(remote_host) + return configured or plat + return plat or ("windows" if IS_WINDOWS else "") + + def _unlink_session_artifacts(session_id: str) -> None: + for pattern in (f"{session_id}.*", f"{session_id}_run.*"): + for path in TMUX_LOG_DIR.glob(pattern): + # Keep the per-session .stop marker — the bash retry loop + # checks it between attempts. + if path.suffix == ".stop": + continue + try: + path.unlink() + except OSError: + pass + + _STOPPED_REPOS_PATH = TMUX_LOG_DIR / "cookbook-stopped-repos.json" + + def _read_stopped_repos() -> set[str]: + try: + text = _STOPPED_REPOS_PATH.read_text(encoding="utf-8").strip() + if not text: + return set() + # Legacy JSON list from earlier builds — still readable on upgrade. + if text.startswith("["): + data = json.loads(text) + if isinstance(data, list): + return {str(x) for x in data if x} + return {line.strip() for line in text.splitlines() if line.strip()} + except Exception: + pass + return set() + + def _write_stopped_repos(repos: set[str]) -> None: + TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True) + _STOPPED_REPOS_PATH.write_text( + "\n".join(sorted(repos)) + ("\n" if repos else ""), + encoding="utf-8", + ) + + def _mark_download_stopped(repo_id: str) -> None: + if not repo_id: + return + repos = _read_stopped_repos() + repos.add(repo_id) + _write_stopped_repos(repos) + + def _clear_download_stopped(repo_id: str) -> None: + if not repo_id: + return + repos = _read_stopped_repos() + if repo_id in repos: + repos.remove(repo_id) + _write_stopped_repos(repos) + + def _session_stop_file(session_id: str) -> Path: + return TMUX_LOG_DIR / f"{session_id}.stop" + + def _write_session_stop_marker(session_id: str, repo_id: str | None = None) -> None: + TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True) + try: + _session_stop_file(session_id).write_text("1", encoding="utf-8") + except OSError: + pass + if repo_id: + _mark_download_stopped(repo_id) + + def _bash_download_stop_guard(session_id: str, repo_id: str | None = None) -> list[str]: + """Bash lines that honour a user stop between download attempts.""" + stop = shlex.quote(_session_stop_file(session_id).as_posix()) + lines = [ + f"_ODYSSEUS_STOP_FILE={stop}", + f"_ODYSSEUS_STOPPED_REPOS={shlex.quote(_STOPPED_REPOS_PATH.as_posix())}", + "trap 'echo \"\"; echo \"DOWNLOAD_STOPPED\"; exit 130' INT TERM", + ] + if repo_id: + lines.append(f"_ODYSSEUS_REPO={shlex.quote(repo_id)}") + return lines + + def _bash_download_attempt_guard() -> str: + return ( + ' if [ -f "$_ODYSSEUS_STOP_FILE" ]; then ' + 'echo ""; echo "DOWNLOAD_STOPPED"; exit 130; fi; ' + 'if [ -n "${_ODYSSEUS_REPO:-}" ] && [ -f "${_ODYSSEUS_STOPPED_REPOS:-}" ] ' + '&& grep -Fxq "$_ODYSSEUS_REPO" "$_ODYSSEUS_STOPPED_REPOS" 2>/dev/null; then ' + 'echo ""; echo "DOWNLOAD_STOPPED"; exit 130; fi' + ) + + def _find_live_local_download(repo_id: str) -> dict | None: + """Live LOCAL download of this repo, if any (honours user-stop markers).""" + if repo_id in _read_stopped_repos(): + live = _probe_live_local_download(repo_id) + if live: + if live.get("session_id"): + _stop_local_windows_session(live["session_id"], repo_id) + elif live.get("orphan_pid") and pid_alive(live["orphan_pid"]): + kill_process_tree(live["orphan_pid"]) + return None + return _probe_live_local_download(repo_id) + + def _probe_live_local_download(repo_id: str) -> dict | None: + """Probe for a live LOCAL download without honouring user-stop markers.""" + try: + sids = sorted({ + p.stem.removesuffix("_run") + for p in TMUX_LOG_DIR.glob("cookbook-*.sh") + }) + except OSError: + return None + for sid in sids: + if not _SESSION_ID_RE.match(sid): + continue + script = TMUX_LOG_DIR / (f"{sid}_run.sh" if IS_WINDOWS else f"{sid}.sh") + try: + script_text = script.read_text(encoding="utf-8") + except OSError: + continue + if not _cmdline_references_hf_repo(script_text, repo_id): + continue + if IS_WINDOWS: + try: + pid = int((TMUX_LOG_DIR / f"{sid}.pid").read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + continue + if pid_alive(pid): + return {"session_id": sid} + else: + try: + probe = subprocess.run( + ["tmux", "has-session", "-t", sid], + capture_output=True, + timeout=5, + ) + except Exception: + continue + if probe.returncode == 0: + return {"session_id": sid} + if IS_WINDOWS: + orphan_pid = _scan_windows_download_processes(repo_id) + if orphan_pid: + return {"orphan_pid": orphan_pid} + return None + def _scan_windows_session_pids(session_id: str) -> list[int] | None: + """PIDs whose command line references this cookbook session's wrappers.""" + if not session_id: + return [] + sid = session_id.replace("'", "''") + ps = ( + "Get-CimInstance Win32_Process | " + f"Where-Object {{ $_.CommandLine -and $_.CommandLine -like '*{sid}*' }} | " + "ForEach-Object { $_.ProcessId }" + ) + try: + probe = subprocess.run( + ["powershell", "-NoProfile", "-Command", ps], + capture_output=True, + text=True, + timeout=15, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except Exception: + return None + if probe.returncode != 0: + return None + out = probe.stdout or "" + pids: list[int] = [] + for line in out.splitlines(): + try: + pids.append(int(line.strip())) + except ValueError: + continue + return pids + + def _kill_local_windows_pid(pid: int) -> bool: + """Kill a Windows process tree and verify that its root exited.""" + try: + result = subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(pid)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=15, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except Exception: + return False + if result.returncode != 0: + return False + for _ in range(10): + if not pid_alive(pid): + return True + time.sleep(0.05) + return not pid_alive(pid) + + def _stop_local_windows_session(session_id: str, repo_id: str | None = None) -> dict: + """Kill a local Windows detached cookbook session and orphan downloaders.""" + pid_path = TMUX_LOG_DIR / f"{session_id}.pid" + found_process = False + detail: list[str] = [] + try: + pid = int(pid_path.read_text(encoding="utf-8").strip()) + if pid_alive(pid): + found_process = True + if not _kill_local_windows_pid(pid): + return { + "ok": False, + "stopped": False, + "error": f"failed to stop local Windows process tree pid {pid}", + "detail": "recovery artifacts preserved", + } + detail.append(f"killed pid {pid}") + except (OSError, ValueError): + pass + for _ in range(3): + scanned_pids = _scan_windows_session_pids(session_id) + if scanned_pids is None: + return { + "ok": False, + "stopped": False, + "error": "failed to scan local Windows session processes", + "detail": "recovery artifacts preserved", + } + session_pids = sorted({p for p in scanned_pids if pid_alive(p)}) + if not session_pids: + break + for pid in session_pids: + found_process = True + if not _kill_local_windows_pid(pid): + return { + "ok": False, + "stopped": False, + "error": f"failed to stop local Windows session process pid {pid}", + "detail": "recovery artifacts preserved", + } + detail.append(f"killed session process pid {pid}") + if repo_id: + for _ in range(3): + scan_ok, orphan = _scan_windows_download_processes_checked(repo_id) + if not scan_ok: + return { + "ok": False, + "stopped": False, + "error": "failed to scan local Windows download processes", + "detail": "recovery artifacts preserved", + } + if not orphan or not pid_alive(orphan): + break + found_process = True + if not _kill_local_windows_pid(orphan): + return { + "ok": False, + "stopped": False, + "error": f"failed to stop local Windows orphan downloader pid {orphan}", + "detail": "recovery artifacts preserved", + } + detail.append(f"killed orphan downloader pid {orphan}") + # Already-dead sessions are success (same as tmux "session not found"): + # scans succeeded and nothing live remains. Only kill/scan failures fail. + _write_session_stop_marker(session_id, repo_id) + _unlink_session_artifacts(session_id) + return { + "ok": True, + "stopped": found_process, + "detail": "; ".join(detail) or "session already gone", + } + + async def _stop_remote_windows_session( + session_id: str, + remote: str, + ssh_port: str | None = None, + repo_id: str | None = None, + ) -> dict: + """PR1: remote Windows stop is out of scope — fail closed.""" + return { + "ok": False, + "stopped": False, + "error": "remote Windows stop-session is not supported in this build", + "detail": "recovery artifacts preserved", + } + + def _tmux_stop_succeeded(returncode: int, stderr: bytes | str = b"") -> bool: + if returncode == 0: + return True + err = ( + stderr.decode("utf-8", errors="replace") + if isinstance(stderr, bytes) + else str(stderr) + ).lower() + return any( + s in err + for s in ("no server running", "can't find session", "session not found") + ) + + async def _stop_cookbook_session_impl( + session_id: str, + remote_host: str = "", + ssh_port: str | None = None, + platform: str = "", + repo_id: str | None = None, + ) -> dict: + if not _SESSION_ID_RE.match(session_id): + return {"ok": False, "error": "invalid session_id"} + remote = (remote_host or "").strip() + sport = ssh_port or "" + is_win = _resolve_windows_platform(remote or None, platform) == "windows" + if remote: + if is_win: + return await _stop_remote_windows_session(session_id, remote, sport, repo_id) + _write_session_stop_marker(session_id, repo_id) + sid = shlex.quote(session_id) + ssh_args = [ + "ssh", + "-o", "ConnectTimeout=5", + "-o", "StrictHostKeyChecking=no", + ] + safe_port = _coerce_ssh_port(sport) + if safe_port and safe_port != "22": + ssh_args.extend(["-p", safe_port]) + ssh_args.append(remote) + ssh_args.append( + f"tmux has-session -t {sid} 2>/dev/null || exit 0; " + f"tmux send-keys -t {sid} C-c 2>/dev/null; " + f"sleep 2; tmux kill-session -t {sid}" + ) + proc = await asyncio.create_subprocess_exec( + *ssh_args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _stdout, stderr = await proc.communicate() + ok = _tmux_stop_succeeded(proc.returncode, stderr) + return {"ok": ok, "exit_code": proc.returncode} + if IS_WINDOWS or is_win: + return await asyncio.to_thread(_stop_local_windows_session, session_id, repo_id) + _write_session_stop_marker(session_id, repo_id) + sid = shlex.quote(session_id) + cmd = ( + f"tmux has-session -t {sid} 2>/dev/null || exit 0; " + f"tmux send-keys -t {sid} C-c 2>/dev/null; " + f"sleep 2; tmux kill-session -t {sid}" + ) + proc = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _stdout, stderr = await proc.communicate() + ok = _tmux_stop_succeeded(proc.returncode, stderr) + return {"ok": ok, "exit_code": proc.returncode} + @router.post("/api/model/download") async def model_download(request: Request, req: ModelDownloadRequest): """Download a HuggingFace model in a tmux session. @@ -1045,6 +1541,46 @@ async def model_download(request: Request, req: ModelDownloadRequest): req.local_dir = _validate_local_dir(req.local_dir) req.hf_token = "" if is_ollama_download else (req.hf_token or _load_stored_hf_token()) _validate_token(req.hf_token) + + if not is_ollama_download: + # Explicit launch from the UI/agent — clear any prior user-stop + # marker so Retry works, while background auto-reattach stays off. + _clear_download_stopped(req.repo_id) + # Concurrent downloads of the same repo deadlock on the HF cache's + # per-file locks ("Still waiting to acquire lock..."). If a live local + # session is already downloading this repo, reattach the UI to it instead + # of launching a duplicate. Remote POSIX hosts are covered by the + # frontend's tmux has-session zombie probe; Ollama serializes pulls in + # its own daemon. + if not is_ollama_download and not (req.remote_host or "").strip(): + live = await asyncio.to_thread(_find_live_local_download, req.repo_id) + if live and live.get("session_id"): + live_sid = live["session_id"] + logger.info( + f"Download of {req.repo_id} already running locally in {live_sid}; reattaching" + ) + return { + "ok": True, + "session_id": live_sid, + "remote": "local", + "reused": True, + } + if live and live.get("orphan_pid"): + pid = live["orphan_pid"] + logger.warning( + f"Download of {req.repo_id} blocked: untracked downloader process " + f"pid={pid} is live locally" + ) + return { + "ok": False, + "error": ( + f"Another process (pid {pid}) is already downloading {req.repo_id} " + "on local outside cookbook tracking — likely left over from an " + "earlier stop. Wait for it to finish, or end its process tree (e.g. " + f"taskkill /F /T /PID {pid}) and retry; the download resumes from cache." + ), + "session_id": "", + } if req.remote_host and not req.env_prefix: req.env_prefix = _server_env_prefix_for_download(req.remote_host) TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True) @@ -1260,12 +1796,14 @@ async def model_download(request: Request, req: ModelDownloadRequest): # download's "not authorized" failure can be told apart from a missing # token (the token is masked — we only print applied / not-set). runner_lines.append(_HF_TOKEN_STATUS_SNIPPET) - # Wrap the download in a retry loop. Large HF/Ollama transfers can - # hit transient network failures; both backends resume cached partials. + # Retry transient failures, but honour explicit user stops between + # attempts so a stopped download cannot silently resume. mw = 4 if req.disable_hf_transfer else 8 + runner_lines.extend(_bash_download_stop_guard(session_id, req.repo_id)) runner_lines.append('_max_retries=10; _attempt=0; _ec=0') runner_lines.append('while [ $_attempt -lt $_max_retries ]; do') runner_lines.append(' _attempt=$((_attempt+1))') + runner_lines.append(_bash_download_attempt_guard()) if is_ollama_download: runner_lines.append(' eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null') else: @@ -1275,6 +1813,7 @@ async def model_download(request: Request, req: ModelDownloadRequest): runner_lines.append(' if [ $_attempt -lt $_max_retries ]; then') runner_lines.append(' echo ""; echo "Download attempt $_attempt failed (exit $_ec) — retrying in 30s..."') runner_lines.append(' sleep 30') + runner_lines.append(_bash_download_attempt_guard()) runner_lines.append(' fi') runner_lines.append('done') runner_lines.append('if [ $_ec -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $_ec after $_attempt attempts)"; fi') @@ -1305,17 +1844,21 @@ async def model_download(request: Request, req: ModelDownloadRequest): # "not authorized" failure apart from a missing token. if not is_ollama_download: lines.append(_HF_TOKEN_STATUS_SNIPPET) - # Retry loop — same rationale as the remote-bash path. Issue #2722. + # Retry transient failures, but honour explicit user stops between + # attempts so a stopped download cannot silently resume. _hf_invoke = 'eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null' if is_ollama_download else (hf_cmd if IS_WINDOWS else f"{hf_cmd} < /dev/null") + lines.extend(_bash_download_stop_guard(session_id, req.repo_id)) lines.append('_max_retries=10; _attempt=0; _ec=0') lines.append('while [ $_attempt -lt $_max_retries ]; do') lines.append(' _attempt=$((_attempt+1))') + lines.append(_bash_download_attempt_guard()) lines.append(f' {_hf_invoke}') lines.append(' _ec=$?') lines.append(' if [ $_ec -eq 0 ]; then break; fi') lines.append(' if [ $_attempt -lt $_max_retries ]; then') lines.append(' echo ""; echo "Download attempt $_attempt failed (exit $_ec) — retrying in 30s..."') lines.append(' sleep 30') + lines.append(_bash_download_attempt_guard()) lines.append(' fi') lines.append('done') lines.append('if [ $_ec -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $_ec after $_attempt attempts)"; fi') @@ -3269,6 +3812,46 @@ async def list_gpus(request: Request, host: str | None = None, ssh_port: str | N return {"ok": False, "error": nvidia_error or "No GPU memory probe available", "gpus": []} + class StopSessionRequest(BaseModel): + session_id: str + remote_host: str | None = None + ssh_port: str | None = None + platform: str | None = None + repo_id: str | None = None + # download | serve | dependency | … — gates download-only stop side effects + task_type: str | None = None + + @router.post("/api/cookbook/stop-session") + async def stop_cookbook_session(request: Request, req: StopSessionRequest): + """Stop a cookbook download/serve session (local or remote, all platforms). + + Centralizes kill logic server-side so the UI does not have to guess + whether a local task is tmux-backed or a Windows detached process tree. + """ + require_admin(request) + validate_remote_host(req.remote_host) + sport = validate_ssh_port(req.ssh_port) + repo_id_raw = (req.repo_id or "").strip() + task_type = (req.task_type or "").strip().lower() + # repo_id is optional download-only metadata for stopped-repo markers and + # orphan downloader cleanup. Never validate it before kill — dependency + # rows store pip labels such as llama-cpp-python[server] in payload.repo_id. + # Serve tasks also carry HF-shaped repo_id; only honour it for downloads. + repo_id_for_stop = ( + repo_id_raw + if task_type == "download" and repo_id_raw and _REPO_ID_RE.match(repo_id_raw) + else None + ) + platform = _resolve_windows_platform(req.remote_host, req.platform) + return await _stop_cookbook_session_impl( + req.session_id.strip(), + remote_host=req.remote_host or "", + ssh_port=sport, + platform=platform, + repo_id=repo_id_for_stop, + ) + + class KillPidRequest(BaseModel): pid: int host: str | None = None diff --git a/src/tools/system.py b/src/tools/system.py index 813d57df2..f44dacdec 100644 --- a/src/tools/system.py +++ b/src/tools/system.py @@ -555,6 +555,7 @@ async def do_api_call(content: str) -> Dict: ("POST", "/api/cookbook/packages/install"), ("POST", "/api/cookbook/rebuild-engine"), ("POST", "/api/cookbook/kill-pid"), + ("POST", "/api/cookbook/stop-session"), # Use the named tools (download_model / serve_model) — they handle # host-name resolution, per-host env_prefix, AND register the task # in cookbook state so it shows in the UI + list_downloads. Hitting @@ -679,6 +680,8 @@ async def do_app_api(content: str, owner: Optional[str] = None) -> Dict: return {"error": "Don't POST /api/cookbook/rebuild-engine via app_api — engine rebuild mutates local or remote host state. Use the dedicated Cookbook UI/flow instead.", "exit_code": 1} if "/api/cookbook/kill-pid" in path: return {"error": "Don't POST /api/cookbook/kill-pid via app_api — process signalling is host control. Use the dedicated Cookbook stop/diagnostic flow instead.", "exit_code": 1} + if "/api/cookbook/stop-session" in path: + return {"error": "Don't POST /api/cookbook/stop-session via app_api — session stop is host process control. Use stop_served_model, cancel_download, or the Cookbook UI stop flow instead.", "exit_code": 1} if "/api/model/download" in path: return {"error": "Don't POST /api/model/download directly — use the `download_model` tool (it resolves the server name, sets the venv env_prefix, and registers the task so it shows in the UI).", "exit_code": 1} if "/api/model/serve" in path: diff --git a/static/js/cookbookRunning.js b/static/js/cookbookRunning.js index 5057d40d5..4fe8641c3 100644 --- a/static/js/cookbookRunning.js +++ b/static/js/cookbookRunning.js @@ -11,6 +11,8 @@ import { computeProgressSignal } from './cookbookProgressSignal.js'; import { portOf, nextFreePort } from './cookbookPorts.js'; import { topPortalZ } from './toolWindowZOrder.js'; +const _RECONNECT_STATUSES = ['running', 'ready', 'loading', 'warming', 'starting']; + // Human-friendly badge label for a task's internal status. Avoids surfacing // the word "error" in the sidebar — a server the user stopped or one that // quit cleanly reads as "stopped", not "error". @@ -52,6 +54,14 @@ function _taskBadge(task) { return { text: _statusLabel(task.status, task.type), cls: 'cookbook-task-' + task.status }; } +function _applyTaskBadge(badge, task, progressText) { + if (!badge) return; + const view = progressText != null ? { ...task, progress: progressText } : task; + const bdg = _taskBadge(view); + badge.textContent = bdg.text; + badge.className = 'cookbook-task-status' + (bdg.cls ? ` ${bdg.cls}` : ''); +} + function _ggufDisplayPartFromPath(path) { const parts = String(path || '').split('/').filter(Boolean); const file = parts[parts.length - 1] || ''; @@ -119,7 +129,7 @@ function _downloadOutputLooksActive(task) { if (!task || task.type !== 'download') return false; const out = task.output || ''; if (!out) return false; - if (out.includes('DOWNLOAD_OK') || out.includes('DOWNLOAD_FAILED')) return false; + if (out.includes('DOWNLOAD_OK') || out.includes('DOWNLOAD_FAILED') || out.includes('DOWNLOAD_STOPPED')) return false; // An active shard line: filename + a colon + a percentage that isn't 100%. // We catch any in-flight shard or "Downloading 'X' to ..." line (no %). return /model-\d+-of-\d+\.[a-z]+:\s+(?!100%)\d+%/i.test(out) @@ -996,6 +1006,117 @@ function _animateOutThenRemove(el, sessionId) { setTimeout(() => _removeTask(sessionId), 360); } +function _applyStoppedTaskCard(el, task) { + const badge = el.querySelector('.cookbook-task-status'); + if (badge) { + badge.textContent = _statusLabel('stopped', task.type); + badge.className = 'cookbook-task-status cookbook-task-stopped'; + } + el.dataset.status = 'stopped'; + const pre = el.querySelector('.cookbook-output-pre'); + const output = pre?.textContent || task.output || ''; + const stamped = task.type === 'download' && !output.includes('DOWNLOAD_STOPPED') + ? (output.trimEnd() ? output.trimEnd() + '\n\n' : '') + 'DOWNLOAD_STOPPED' + : output; + if (pre && stamped !== output) pre.textContent = stamped; + _updateTask(task.sessionId, { status: 'stopped', _userStopped: true, output: stamped || output }); + const wave = el.querySelector('.cookbook-task-wave'); + if (wave) wave.style.display = 'none'; + const uptime = el.querySelector('.cookbook-task-uptime'); + if (uptime) { + uptime.style.display = 'none'; + if (el._uptimeInterval) { + clearInterval(el._uptimeInterval); + el._uptimeInterval = null; + } + } + const startNow = el.querySelector('.cookbook-task-start-now'); + if (startNow) startNow.style.display = 'none'; + const check = el.querySelector('.cookbook-task-check'); + if (check) check.style.display = ''; +} + +async function _executeTaskStop(el, task) { + if (el._abort) el._abort.abort(); + const badge = el.querySelector('.cookbook-task-status'); + if (badge) { + badge.textContent = 'stopping...'; + badge.className = 'cookbook-task-status cookbook-task-stopping'; + } + _updateTask(task.sessionId, { _userStopped: true }); + const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''; + if (task.type === 'serve' && task.payload) { + _removeEndpointByUrl(_endpointUrlForTask(task, outputText)); + } + const ollamaUnload = _ollamaUnloadCommand(task, outputText); + if (ollamaUnload) { + try { + await fetch('/api/shell/exec', { + method: 'POST', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command: ollamaUnload }), + }); + } catch {} + } + try { + const result = await _stopCookbookSession(task); + return !!(result && result.ok); + } catch { + return false; + } +} + +async function _onTaskStop(el, task, { removeAfter = false } = {}) { + const priorStatus = task.status; + const wasActiveServe = task.type === 'serve' && _RECONNECT_STATUSES.includes(priorStatus); + const wasActiveDownload = task.type === 'download' && priorStatus === 'running'; + const badge = el.querySelector('.cookbook-task-status'); + const stopOk = await _executeTaskStop(el, task); + if (!stopOk) { + try { uiModule.showToast('Stop failed — download may still be running in the background', 'error'); } catch (_) {} + if (wasActiveServe || wasActiveDownload) { + if (badge) _applyTaskBadge(badge, task); + el.dataset.status = priorStatus; + _updateTask(task.sessionId, { _userStopped: false, status: priorStatus }); + _reconnectTask(el, task); + } + return false; + } + if (removeAfter) { + _animateOutThenRemove(el, task.sessionId); + } else { + _applyStoppedTaskCard(el, task); + } + return true; +} + +async function _stopCookbookSession(task) { + const taskType = task?.type || ''; + const body = { + session_id: task.sessionId, + remote_host: task.remoteHost || '', + ssh_port: _getPort(task) || '', + platform: task.platform || _getPlatform(task) || '', + task_type: taskType, + }; + if (taskType === 'download') { + const repoId = task?.payload?.repo_id || task?.payload?.repoId || ''; + if (repoId) body.repo_id = repoId; + } + try { + const r = await fetch('/api/cookbook/stop-session', { + method: 'POST', + credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!r.ok) return { ok: false }; + return await r.json(); + } catch { + return { ok: false }; + } +} + // ── tmux / Windows session commands ── function _taskRemoteHost(task) { @@ -1007,7 +1128,9 @@ function _remoteTmuxPrefix() { } export function _tmuxCmd(task, tmuxArgs) { - if (_isWindows(task)) { + const localWin = !_taskRemoteHost(task) + && (_isWindows(task) || _isWindows('local')); + if (_isWindows(task) || localWin) { return _winSessionCmd(task, tmuxArgs); } const host = _taskRemoteHost(task); @@ -1061,8 +1184,8 @@ function _winSessionStopTreePs(task) { const sid = task.sessionId; const stopTree = `function Stop-Tree([int]$Id) { Get-CimInstance Win32_Process -Filter ('ParentProcessId = ' + $Id) -ErrorAction SilentlyContinue | ForEach-Object { Stop-Tree ([int]$_.ProcessId) }; Stop-Process -Id $Id -Force -ErrorAction SilentlyContinue }`; return host - ? `${stopTree}; $p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue` - : `${stopTree}; $p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`; + ? `${stopTree}; $p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }` + : `${stopTree}; $p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p -match '^\\d+$') { Stop-Tree ([int]$p) }`; } export function _tmuxGracefulKill(task) { @@ -2866,53 +2989,32 @@ export function _renderRunningTab() { _reconnectTask(el, task); }); - // Wire stop + // Wire stop — server-side stop-session; rollback on failure for live tasks. el.querySelector('.cookbook-task-action-stop').addEventListener('click', async () => { - // Abort the reconnect loop before sending kill so that a DOWNLOAD_FAILED - // marker written by the shell wrapper (on SIGINT/non-zero exit) cannot - // trigger an auto-retry after a manual stop. - if (el._abort) el._abort.abort(); - const badge = el.querySelector('.cookbook-task-status'); - if (badge) { badge.textContent = 'stopping...'; badge.className = 'cookbook-task-status cookbook-task-stopping'; } - el.dataset.status = 'stopped'; - _updateTask(task.sessionId, { _userStopped: true }); - const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''; - // Drop the model endpoint so the picker stops listing it. - if (task.type === 'serve' && task.payload) { - _removeEndpointByUrl(_endpointUrlForTask(task, outputText)); - } - const ollamaUnload = _ollamaUnloadCommand(task, outputText); - if (ollamaUnload) { - try { - await fetch('/api/shell/exec', { - method: 'POST', credentials: 'same-origin', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: ollamaUnload }), - }); - } catch {} - } - // Gracefully stop (C-c, then kill the session) so it's fully down... - try { - await fetch('/api/shell/exec', { - method: 'POST', credentials: 'same-origin', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: _tmuxGracefulKill(task) }), - }); - } catch {} - // ...then smoothly fade/slide the card out and auto-remove it — no manual - // ⋮ → Remove needed. - _animateOutThenRemove(el, task.sessionId); + const liveTask = _loadTasks().find(t => t.sessionId === task.sessionId) || task; + await _onTaskStop(el, liveTask, { removeAfter: false }); }); - // Wire kill — awaits the SSH/tmux kill and verifies the session is - // actually gone before removing the row. Previously fire-and-forget, - // which meant a failed kill (wrong remoteHost, SSH error, tmux server - // already exited) silently left the live serve running while the - // row disappeared from the UI. + // Wire kill — running tasks: stop then dismiss; finished/stopped: remove only. el.querySelector('.cookbook-task-action-kill').addEventListener('click', async () => { - const outputText = el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''; - const isLive = task.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(task.status || ''); - const ollamaUnload = _ollamaUnloadCommand(task, outputText); + const liveTask = _loadTasks().find(t => t.sessionId === task.sessionId) || task; + const liveStatus = liveTask.status || el.dataset.status || task.status; + const _isLiveServe = liveTask.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(liveStatus || ''); + const _isActive = _isLiveServe || (liveStatus === 'running' && liveTask.type === 'download'); + if (_isActive) { + await _onTaskStop(el, liveTask, { removeAfter: true }); + return; + } + // Inactive download rows are UI history only — the session is already dead. + // stop-session with repo_id would mark the model user-stopped and kill any + // newer Restart of the same repo (duplicate-card cleanup scenario). + if (liveTask.type === 'download') { + _animateOutThenRemove(el, liveTask.sessionId); + return; + } + const outputText = el.querySelector('.cookbook-output-pre')?.textContent || liveTask.output || ''; + const isLive = liveTask.type === 'serve' && ['running', 'ready', 'loading', 'warming', 'starting'].includes(liveStatus || ''); + const ollamaUnload = _ollamaUnloadCommand(liveTask, outputText); if (ollamaUnload) { try { await fetch('/api/shell/exec', { @@ -2924,41 +3026,30 @@ export function _renderRunningTab() { } let killOk = true; try { - const r = await fetch('/api/shell/exec', { - method: 'POST', credentials: 'same-origin', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: _tmuxGracefulKill(task) }), - }); - if (r.ok) { - const out = await r.json(); - // Don't trust exit_code alone — tmux kill returns 0 even when - // there was nothing to kill. Verify the session is actually gone. - if (task.sessionId && isLive) { - try { - const probe = await fetch('/api/shell/exec', { - method: 'POST', credentials: 'same-origin', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ command: _tmuxCmd(task, `has-session -t ${task.sessionId}`) }), - }); - if (probe.ok) { - const pj = await probe.json(); - // has-session exits 0 when session STILL exists; non-zero = gone. - if ((pj.exit_code || 0) === 0) killOk = false; - } - } catch (_) { /* probe best-effort; trust kill */ } - } - } else { - killOk = false; + const result = await _stopCookbookSession(liveTask); + killOk = !!(result && result.ok); + if (killOk && liveTask.sessionId && isLive && !_isWindows(liveTask)) { + try { + const probe = await fetch('/api/shell/exec', { + method: 'POST', credentials: 'same-origin', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ command: _tmuxCmd(liveTask, `has-session -t ${liveTask.sessionId}`) }), + }); + if (probe.ok) { + const pj = await probe.json(); + if ((pj.exit_code || 0) === 0) killOk = false; + } + } catch (_) { /* probe best-effort; trust stop-session */ } } } catch (_) { killOk = false; } if (!killOk) { try { uiModule.showToast('Kill failed — session may still be running. Check `tmux ls` on the server.', 'error'); } catch (_) {} - return; // leave the row so the user can retry + return; } - if (task.type === 'serve' && task.payload) { - const endpointUrl = _endpointUrlForTask(task, outputText); + if (liveTask.type === 'serve' && liveTask.payload) { + const endpointUrl = _endpointUrlForTask(liveTask, outputText); _removeEndpointByUrl(endpointUrl); - const modelName = task.payload.model || task.name || ''; + const modelName = liveTask.payload.model || liveTask.name || ''; if (modelName) { fetch('/api/model-endpoints', { credentials: 'same-origin' }) .then(r => r.json()) @@ -2968,7 +3059,7 @@ export function _renderRunningTab() { }).catch(() => {}); } } - _animateOutThenRemove(el, task.sessionId); + _animateOutThenRemove(el, liveTask.sessionId); }); // Wire retry @@ -3481,6 +3572,13 @@ async function _reconnectTask(el, task) { badge.textContent = 'finishing'; badge.className = 'cookbook-task-status cookbook-task-running'; } + if (snapshot.includes('DOWNLOAD_STOPPED')) { + badge.textContent = _statusLabel('stopped', task.type); + badge.className = 'cookbook-task-status cookbook-task-stopped'; + _updateTask(task.sessionId, { status: 'stopped', _userStopped: true }); + el.dataset.status = 'stopped'; + break; + } if (snapshot.includes('DOWNLOAD_FAILED')) { // The wrapper prints DOWNLOAD_FAILED but exits 0, and per-file // "Download complete"/"100%" lines make it look successful — so @@ -3491,7 +3589,7 @@ async function _reconnectTask(el, task) { const _accessDenied = /Access to model.*is restricted|gated repo|GatedRepoError|401 Unauthorized|403 Forbidden|not in the authorized list|awaiting a review|must (?:be authenticated|have access)/i.test(snapshot); const _dlKey = task.payload?.repo_id || task.name; const _dlN = _dlRetryCount.get(_dlKey) || 0; - if (!controller.signal.aborted && !_accessDenied && task.type === 'download' && task.payload && _dlN < _DL_MAX_AUTO_RETRY) { + if (!controller.signal.aborted && !task._userStopped && !_accessDenied && task.type === 'download' && task.payload && _dlN < _DL_MAX_AUTO_RETRY) { // Auto-retry: kill the dead session and re-launch (resumes from // the cached .incomplete files) after a short delay. _dlRetryCount.set(_dlKey, _dlN + 1); diff --git a/tests/test_cookbook_download_restart_js.py b/tests/test_cookbook_download_restart_js.py new file mode 100644 index 000000000..ab701b19b --- /dev/null +++ b/tests/test_cookbook_download_restart_js.py @@ -0,0 +1,52 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +RUNNING_JS = ROOT / "static" / "js" / "cookbookRunning.js" + + +def test_inactive_download_remove_skips_stop_session(): + source = RUNNING_JS.read_text(encoding="utf-8") + idx = source.index("Inactive download rows are UI history only") + block = source[idx:idx + 500] + assert "liveTask.type === 'download'" in block + assert "_animateOutThenRemove(el, liveTask.sessionId)" in block + assert "_stopCookbookSession(task)" not in block + + +def test_menu_kill_handler_uses_refreshed_task_not_stale_closure(): + source = RUNNING_JS.read_text(encoding="utf-8") + idx = source.index(".cookbook-task-action-kill').addEventListener('click'") + block = source[idx:idx + 900] + assert "const liveTask = _loadTasks().find(t => t.sessionId === task.sessionId)" in block + assert "const liveStatus = liveTask.status || el.dataset.status" in block + assert "await _onTaskStop(el, liveTask" in block + + +def test_download_stopped_is_terminal_for_active_output(): + source = RUNNING_JS.read_text(encoding="utf-8") + idx = source.index("function _downloadOutputLooksActive") + block = source[idx:idx + 450] + assert "DOWNLOAD_STOPPED" in block + + +def test_failed_stop_rolls_back_live_serve_and_download_statuses(): + source = RUNNING_JS.read_text(encoding="utf-8") + idx = source.index("async function _onTaskStop") + block = source[idx:idx + 1000] + assert "_RECONNECT_STATUSES.includes(priorStatus)" in block + assert "task.type === 'download' && priorStatus === 'running'" in block + failed = block[block.index("if (!stopOk)"):block.index("if (removeAfter)")] + assert "el.dataset.status = priorStatus" in block + assert "_userStopped: false, status: priorStatus" in failed + assert "_applyStoppedTaskCard" not in failed + + +def test_stop_cookbook_session_sends_repo_id_only_for_downloads(): + source = RUNNING_JS.read_text(encoding="utf-8") + idx = source.index("async function _stopCookbookSession") + block = source[idx:idx + 900] + assert "task_type: taskType" in block + assert "if (taskType === 'download')" in block + assert "body.repo_id = repoId" in block + assert block.index("if (taskType === 'download')") < block.index("body.repo_id = repoId") diff --git a/tests/test_cookbook_helpers.py b/tests/test_cookbook_helpers.py index bf6c47d4b..8e2fb8017 100644 --- a/tests/test_cookbook_helpers.py +++ b/tests/test_cookbook_helpers.py @@ -724,9 +724,13 @@ def test_local_windows_download_pid_tracks_inner_bash_and_stop_kills_tree(): running_src = (Path(__file__).resolve().parents[1] / "static" / "js" / "cookbookRunning.js").read_text(encoding="utf-8") assert 'printf \'%s\\\\n\' \\"$$\\" > {pp}' in routes_src + assert "/api/cookbook/stop-session" in routes_src + assert "_scan_windows_session_pids" in routes_src + assert "_cmdline_references_hf_repo" in routes_src + assert "grep -Fxq" in routes_src assert "function Stop-Tree([int]$Id)" in running_src - assert "('ParentProcessId = ' + $Id)" in running_src - assert "Stop-Tree ([int]$p)" in running_src + assert "_stopCookbookSession" in running_src + assert "_winSessionStopTreePs" in running_src def test_llama_cpp_rebuild_cmd_runs_clean_on_a_fresh_home(tmp_path): diff --git a/tests/test_cookbook_local_windows_stop.py b/tests/test_cookbook_local_windows_stop.py new file mode 100644 index 000000000..5dd9f42c2 --- /dev/null +++ b/tests/test_cookbook_local_windows_stop.py @@ -0,0 +1,217 @@ +"""Local-Windows cookbook stop-session / orphan guard tests (PR1 scope).""" +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest +from starlette.requests import Request + +import routes.cookbook_routes as cookbook_routes +from routes.cookbook_routes import _cmdline_references_hf_repo, _coerce_ssh_port + + +def _route_endpoint(path: str, method: str): + router = cookbook_routes.setup_cookbook_routes() + return next( + route.endpoint + for route in router.routes + if route.path == path and method in route.methods + ) + + +def _admin_request(path: str) -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": path, + "headers": [], + "state": {}, + } + ) + + +def test_cmdline_references_hf_repo_exact_match_not_prefix(): + short = "org/model" + long_repo = "org/model-large" + assert _cmdline_references_hf_repo(f"python hf_download.py {short}", short) + assert _cmdline_references_hf_repo(f"hf download {short} --local-dir /tmp", short) + assert _cmdline_references_hf_repo( + f"snapshot_download('{short}', local_dir='/tmp')", short + ) + assert not _cmdline_references_hf_repo(f"python hf_download.py {long_repo}", short) + assert not _cmdline_references_hf_repo(f"hf download {long_repo}", short) + + +def test_coerce_ssh_port_rejects_out_of_range(): + assert _coerce_ssh_port("2222") == "2222" + assert _coerce_ssh_port("0") is None + assert _coerce_ssh_port("65536") is None + + +def test_local_windows_stop_session_wiring_source(): + routes_src = Path(cookbook_routes.__file__).read_text(encoding="utf-8") + running_src = (Path(__file__).resolve().parents[1] / "static" / "js" / "cookbookRunning.js").read_text( + encoding="utf-8" + ) + assert "/api/cookbook/stop-session" in routes_src + assert "_scan_windows_session_pids" in routes_src + assert "_cmdline_references_hf_repo" in routes_src + assert "grep -Fxq" in routes_src + assert "_stopCookbookSession" in running_src + assert "async function _onTaskStop" in running_src + assert "_userStopped: false, status: priorStatus" in running_src + + +@pytest.mark.asyncio +async def test_stop_session_dependency_pip_label_still_kills(monkeypatch): + import asyncio + + endpoint = _route_endpoint("/api/cookbook/stop-session", "POST") + kill_ran: list[bool] = [] + + class FakeProc: + returncode = 0 + + async def communicate(self): + return b"", b"" + + async def fake_subprocess_shell(*args, **kwargs): + kill_ran.append(True) + return FakeProc() + + monkeypatch.setattr(cookbook_routes, "IS_WINDOWS", False) + monkeypatch.setattr(asyncio, "create_subprocess_shell", fake_subprocess_shell) + monkeypatch.setattr(cookbook_routes, "require_admin", lambda request: None) + + req = SimpleNamespace( + session_id="cookbook-deadbeef", + remote_host=None, + ssh_port=None, + platform=None, + repo_id="llama-cpp-python[server]", + task_type="dependency", + ) + result = await endpoint(_admin_request("/api/cookbook/stop-session"), req) + assert result["ok"] is True + assert kill_ran + + +@pytest.mark.asyncio +async def test_stop_session_serve_skips_download_repo_side_effects(monkeypatch): + monkeypatch.setattr(cookbook_routes, "require_admin", lambda request: None) + endpoint = _route_endpoint("/api/cookbook/stop-session", "POST") + seen: list = [] + + async def fake_impl(session_id, remote_host="", ssh_port=None, platform="", repo_id=None): + seen.append({"repo_id": repo_id, "platform": platform}) + return {"ok": True} + + for name, cell in zip(endpoint.__code__.co_freevars, endpoint.__closure__ or ()): + if name == "_stop_cookbook_session_impl": + cell.cell_contents = fake_impl + break + + request = _admin_request("/api/cookbook/stop-session") + req_serve = SimpleNamespace( + session_id="serve-deadbeef", + remote_host=None, + ssh_port=None, + platform=None, + repo_id="org/model", + task_type="serve", + ) + result = await endpoint(request, req_serve) + assert result["ok"] is True + assert seen and seen[0]["repo_id"] is None + + req_dl = SimpleNamespace( + session_id="cookbook-deadbeef", + remote_host=None, + ssh_port=None, + platform=None, + repo_id="org/model", + task_type="download", + ) + await endpoint(request, req_dl) + assert seen[-1]["repo_id"] == "org/model" + + +def test_local_download_guard_present_without_remote_windows(): + src = Path(cookbook_routes.__file__).read_text(encoding="utf-8") + assert "_find_live_local_download" in src + assert "_clear_download_stopped" in src + assert "_find_live_remote_windows_download" not in src + + +@pytest.mark.asyncio +async def test_local_windows_stop_failure_preserves_recovery_state(monkeypatch, tmp_path): + monkeypatch.setattr(cookbook_routes, "TMUX_LOG_DIR", tmp_path) + monkeypatch.setattr( + cookbook_routes, "COOKBOOK_STATE_FILE", str(tmp_path / "state.json") + ) + monkeypatch.setattr(cookbook_routes, "require_admin", lambda request: None) + monkeypatch.setattr(cookbook_routes, "pid_alive", lambda pid: True) + runner = tmp_path / "cookbook-deadbeef_run.ps1" + pid_file = tmp_path / "cookbook-deadbeef.pid" + runner.write_text("hf download org/model", encoding="utf-8") + pid_file.write_text("1234", encoding="utf-8") + + def failed_taskkill(args, **kwargs): + assert args[0] == "taskkill" + return SimpleNamespace(returncode=1, stdout="") + + monkeypatch.setattr(cookbook_routes.subprocess, "run", failed_taskkill) + endpoint = _route_endpoint("/api/cookbook/stop-session", "POST") + req = SimpleNamespace( + session_id="cookbook-deadbeef", + remote_host=None, + ssh_port=None, + platform="windows", + repo_id="org/model", + task_type="download", + ) + + result = await endpoint(_admin_request("/api/cookbook/stop-session"), req) + + assert result["ok"] is False + assert result["stopped"] is False + assert "failed to stop local Windows process tree" in result["error"] + assert runner.exists() + assert pid_file.exists() + assert not (tmp_path / "cookbook-deadbeef.stop").exists() + assert not (tmp_path / "cookbook-stopped-repos.json").exists() + + +@pytest.mark.asyncio +async def test_local_windows_already_gone_stop_is_success(monkeypatch, tmp_path): + monkeypatch.setattr(cookbook_routes, "TMUX_LOG_DIR", tmp_path) + monkeypatch.setattr( + cookbook_routes, "COOKBOOK_STATE_FILE", str(tmp_path / "state.json") + ) + monkeypatch.setattr(cookbook_routes, "require_admin", lambda request: None) + monkeypatch.setattr(cookbook_routes, "pid_alive", lambda pid: False) + runner = tmp_path / "cookbook-deadbeef_run.sh" + runner.write_text("hf download org/model", encoding="utf-8") + + def empty_scan(args, **kwargs): + return SimpleNamespace(returncode=0, stdout="") + + monkeypatch.setattr(cookbook_routes.subprocess, "run", empty_scan) + endpoint = _route_endpoint("/api/cookbook/stop-session", "POST") + req = SimpleNamespace( + session_id="cookbook-deadbeef", + remote_host=None, + ssh_port=None, + platform="windows", + repo_id="org/model", + task_type="download", + ) + + result = await endpoint(_admin_request("/api/cookbook/stop-session"), req) + + assert result["ok"] is True + assert result["stopped"] is False + assert not runner.exists() + assert (tmp_path / "cookbook-deadbeef.stop").exists() From 68191e16c63a28ebea0ae50183418e089c6a11cf Mon Sep 17 00:00:00 2001 From: zakrepashe Date: Thu, 23 Jul 2026 22:34:53 +0300 Subject: [PATCH 2/2] fix(agent): import Any used by tool-event helpers Current dev annotates agent_loop helpers with dict[str, Any] but never imports Any, so launch-windows.ps1 / uvicorn crashes on import. Co-authored-by: Cursor --- src/agent_loop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent_loop.py b/src/agent_loop.py index 592ebaec1..368ad3211 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -12,7 +12,7 @@ import re import time import logging -from typing import AsyncGenerator, List, Dict, Optional, Set +from typing import Any, AsyncGenerator, Dict, List, Optional, Set from urllib.parse import urlparse from src.llm_core import (