Keep cold-start memory setup off the HTTP path

Author: gitcommit90

CHANGELOG.md

@@ -45,6 +45,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Fresh hosts now open the 1Helm server while the optional Mnemosyne Python
+  and embedding runtime is prepared in the background, instead of making
+  first launch and health checks wait on virtual-environment package installs.
 - Newly created and newly assigned skills now appear immediately in already
   open Arsenal and Channel Settings views without a page refresh.
 - Linux upgrades migrate existing compatibility computer records to LXC while

src/server/index.ts

@@ -59,7 +59,7 @@ import { auditEvents, verifyAuditChain } from "./audit.ts";
 import { configurePhoton, mapPhotonChannel, photonStatus, registerPhotonDispatcher, startPhotonConnector, stopPhotonConnector } from "./photon.ts";
 import { photonSetupStatus, startPhotonSetup } from "./photon-auth.ts";
 import { gmailConnectionStatus, saveGmailOAuthClient, startGmailConnection } from "./gmail.ts";
-import { ensureAgentMemory, mnemosyneAvailable, prepareMnemosyneRuntime } from "./memory.ts";
+import { cancelMnemosyneRuntimePreparation, ensureAgentMemory, mnemosyneAvailable, prepareMnemosyneRuntime } from "./memory.ts";
 import { runImprovementPass, scheduleAgentReview, startImprovementLoop } from "./improvements.ts";
 import { runThreadAuditPass, startThreadAuditLoop } from "./thread-audit.ts";
 import { startFollowupLoop, threadFollowupView, bumpThreadFollowup } from "./followups.ts";
@@ -1872,7 +1872,7 @@ async function bootstrap(): Promise<void> {
   registerPhotonDispatcher((bot, channelId, triggerId, threadRootId) => runBot(bot, channelId, triggerId, threadRootId, true));
   registerWorkflowDispatcher((bot, channelId, triggerId, threadRootId) => runBot(bot, channelId, triggerId, threadRootId, true));
   reactivateComputersAfterPreparedRemoval();
-  prepareMnemosyneRuntime();
+  const memoryRuntime = prepareMnemosyneRuntime();
   await startRoutingEngine((activity, ownerUserId) => {
     if (ownerUserId) sendToUsers([ownerUserId], { type: "routing_activity", activity });
     else broadcastAdmins({ type: "routing_activity", activity });
@@ -1908,6 +1908,13 @@ async function bootstrap(): Promise<void> {
     const address = server.address();
     const port = typeof address === "object" && address ? address.port : PORT;
     console.log(`1Helm on 1Helm → http://${HOST === "0.0.0.0" ? "localhost" : HOST}:${port}  (local agent on ${agentPort})  data: ${DATA_DIR}`);
+    void memoryRuntime.then((ready) => {
+      if (!ready) return;
+      for (const channel of q("SELECT id FROM channels WHERE kind='channel' AND status<>'deleted'")) {
+        const agent = agentForChannel(Number(channel.id));
+        if (agent) ensureAgentMemory(agent);
+      }
+    }).catch((error) => console.warn(`1Helm could not prepare durable memory: ${(error as Error).message}`));
     void queueLinuxHostContractMigration(DATA_DIR).catch((error) => console.warn(`1Helm could not queue its Linux host-contract migration: ${(error as Error).message}`));
   });
 }
@@ -1917,6 +1924,7 @@ let shuttingDown = false;
 const shutdown = async (forNativeUpdate = false): Promise<void> => {
   if (shuttingDown) return;
   shuttingDown = true;
+  cancelMnemosyneRuntimePreparation();
   await stopRoutingEngine().catch(() => undefined);
   stopAllConnectors();
   await stopPhotonConnector().catch(() => undefined);

src/server/memory.ts

@@ -1,4 +1,5 @@
-import { execFileSync, spawnSync } from "node:child_process";
+import { execFile, execFileSync, spawnSync } from "node:child_process";
+import { promisify } from "node:util";
 import { existsSync, mkdirSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { DATA_DIR, q1, type Row } from "./db.ts";
@@ -9,6 +10,11 @@ const CONFIG_DIR = join(DATA_DIR, "mnemosyne-runtime", "config");
 const MNEMOSYNE_VERSION = "3.14.0";
 let validatedPython: string | null | undefined;
 let validatedSemanticPython: string | undefined;
+let preparation: Promise<boolean> | null = null;
+let preparationAbort = new AbortController();
+const execFileAsync = promisify(execFile);
+
+const asyncExecOptions = (timeout: number) => ({ timeout, windowsHide: true, signal: preparationAbort.signal });
 
 function hasPinnedRuntime(candidate: string): boolean {
   if (!candidate || !existsSync(candidate)) return false;
@@ -147,7 +153,7 @@ export function recallTranscriptForAgent(agent: Row, query: string, topK = 24):
 export function mnemosyneAvailable(): boolean { return Boolean(pythonRuntime()); }
 
 /** Install the pinned local-first memory runtime into the data root on a fresh 1Helm host. */
-export function prepareMnemosyneRuntime(): boolean {
+async function prepareMnemosyneRuntimeUnlocked(): Promise<boolean> {
   const managedPython = join(DATA_DIR, "mnemosyne-runtime", "venv", "bin", "python");
   const current = pythonRuntime();
   if (current && hasPinnedSemanticRuntime(current)) return true;
@@ -161,9 +167,10 @@ export function prepareMnemosyneRuntime(): boolean {
   // memory instead of destroying a working durable-memory runtime.
   if (current === managedPython) {
     try {
-      execFileSync(current, ["-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--ignore-requires-python", `mnemosyne-memory[embeddings]==${MNEMOSYNE_VERSION}`], { timeout: 600_000, stdio: "ignore" });
+      await execFileAsync(current, ["-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--ignore-requires-python", `mnemosyne-memory[embeddings]==${MNEMOSYNE_VERSION}`], asyncExecOptions(600_000));
       if (hasPinnedSemanticRuntime(current)) return true;
     } catch (error) {
+      if (preparationAbort.signal.aborted) return false;
       console.warn(`Could not add local semantic retrieval to the existing Mnemosyne runtime:`, (error as Error).message);
     }
     return true;
@@ -181,21 +188,27 @@ export function prepareMnemosyneRuntime(): boolean {
     ...(process.platform === "darwin" ? ["/usr/bin/python3"] : []),
   ].filter(Boolean))];
   for (const python of installers) {
+    if (preparationAbort.signal.aborted) break;
     // A failed venv or pip run may still leave an executable Python behind.
     // Replace only this app-managed runtime after proving it cannot import the
     // pinned package; agent databases and all other Application Support remain.
     if (existsSync(venv)) rmSync(venv, { recursive: true, force: true });
     validatedSemanticPython = undefined;
     try {
-      execFileSync(python, ["-m", "venv", venv], { timeout: 60_000, stdio: "ignore" });
+      await execFileAsync(python, ["-m", "venv", venv], asyncExecOptions(60_000));
       try {
-        execFileSync(join(venv, "bin", "python"), ["-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--ignore-requires-python", `mnemosyne-memory[embeddings]==${MNEMOSYNE_VERSION}`], { timeout: 600_000, stdio: "ignore" });
+        const requirement = process.env.NODE_ENV === "test"
+          ? `mnemosyne-memory==${MNEMOSYNE_VERSION}`
+          : `mnemosyne-memory[embeddings]==${MNEMOSYNE_VERSION}`;
+        await execFileAsync(join(venv, "bin", "python"), ["-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--ignore-requires-python", requirement], asyncExecOptions(600_000));
       } catch {
-        execFileSync(join(venv, "bin", "python"), ["-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--ignore-requires-python", `mnemosyne-memory==${MNEMOSYNE_VERSION}`], { timeout: 180_000, stdio: "ignore" });
+        if (preparationAbort.signal.aborted) break;
+        await execFileAsync(join(venv, "bin", "python"), ["-m", "pip", "install", "--disable-pip-version-check", "--no-input", "--ignore-requires-python", `mnemosyne-memory==${MNEMOSYNE_VERSION}`], asyncExecOptions(180_000));
       }
       validatedPython = undefined;
       if (pythonRuntime()) return true;
     } catch (error) {
+      if (preparationAbort.signal.aborted) break;
       console.warn(`Could not prepare Mnemosyne runtime with ${python}:`, (error as Error).message);
     }
   }
@@ -206,3 +219,16 @@ export function prepareMnemosyneRuntime(): boolean {
   validatedSemanticPython = undefined;
   return false;
 }
+
+/** Prepare the optional Python runtime without blocking the HTTP server's
+ * event loop on venv creation or package downloads. Concurrent callers share
+ * one installation attempt. */
+export function prepareMnemosyneRuntime(): Promise<boolean> {
+  if (!preparation) preparation = prepareMnemosyneRuntimeUnlocked();
+  return preparation;
+}
+
+/** Stop only the in-flight app-managed runtime installer during host shutdown. */
+export function cancelMnemosyneRuntimePreparation(): void {
+  preparationAbort.abort();
+}

test/desktop.mjs

@@ -141,6 +141,10 @@ test("desktop entrypoint keeps the renderer sandboxed and data on the Mac", asyn
   assert.match(memoryRuntime, /--ignore-requires-python/);
   assert.match(memoryRuntime, /rmSync\(venv, \{ recursive: true, force: true \}\)/, "an invalid partial app-managed memory venv is repaired without touching agent databases");
   assert.match(memoryRuntime, /process\.platform === "darwin" \? \["\/usr\/bin\/python3"\]/, "macOS retries its bundled Python when a preferred interpreter cannot create the app-managed memory runtime");
+  assert.match(memoryRuntime, /export function prepareMnemosyneRuntime\(\): Promise<boolean>/, "fresh-host memory installation is asynchronous instead of blocking application startup");
+  assert.match(memoryRuntime, /export function cancelMnemosyneRuntimePreparation\(\)/, "host shutdown cancels an in-flight app-managed memory installation");
+  const serverRuntime = await readFile(join(root, "src", "server", "index.ts"), "utf8");
+  assert.match(serverRuntime, /const memoryRuntime = prepareMnemosyneRuntime\(\);[\s\S]*server\.listen\([\s\S]*memoryRuntime\.then/, "the HTTP server becomes ready before optional memory installation and initializes agent databases afterward");
   const memoryBridge = await readFile(join(root, "scripts", "mnemosyne-bridge.py"), "utf8");
   assert.match(memoryBridge, /sys\.version_info < \(3, 10\)/);
   assert.match(memoryBridge, /zip_longest/);

test/feedback-browser.mjs

@@ -49,6 +49,7 @@ test("Feedback button saves a real report and the admin inbox shows it", async (
       ...process.env,
       CTRL_DATA_DIR: dataDir,
       PORT: String(appPort),
+      NODE_ENV: "test",
       HELM_CHANNEL_COMPUTER_BACKEND: "native",
       HELM_FEEDBACK_URL: `http://127.0.0.1:${collectorPort}/v1/feedback`,
       IMPROVEMENT_INTERVAL_MS: "600000",
@@ -60,6 +61,10 @@ test("Feedback button saves a real report and the admin inbox shows it", async (
     await browser.close().catch(() => undefined);
     app.kill("SIGTERM");
     provider.kill("SIGTERM");
+    await Promise.race([
+      new Promise((resolve) => app.once("exit", resolve)),
+      new Promise((resolve) => setTimeout(resolve, 3_000)),
+    ]);
     await new Promise((resolve) => collector.close(resolve));
     rmSync(dataDir, { recursive: true, force: true });
   });