fix(capture): let vision captioning authenticate the way a server can

packages/cli/src/capture/contentExtractor.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -8,13 +8,54 @@ import {
   type VisionCaptionOutcome,
 } from "./contentExtractor.js";
 
-const { generateContentMock } = vi.hoisted(() => ({
+const { generateContentMock, clientOptions, sharpState } = vi.hoisted(() => ({
   generateContentMock: vi.fn(),
+  // How the SDK client was constructed is the whole difference between the Vertex and API-key
+  // paths, so the Vertex cases assert on it rather than on the request.
+  clientOptions: [] as Record<string, unknown>[],
+  // A native abort inside libvips cannot be caught, so the only defence is never running two
+  // renders at once. That is a property of the loop, and this records it.
+  sharpState: {
+    inFlight: 0,
+    maxInFlight: 0,
+    concurrencyCalls: [] as number[],
+    renders: [] as string[],
+  },
 }));
 
+vi.mock("sharp", () => {
+  const pipeline = (filePath: string) => {
+    const chain = {
+      resize: () => chain,
+      flatten: () => chain,
+      png: () => chain,
+      toBuffer: async () => {
+        sharpState.inFlight += 1;
+        sharpState.maxInFlight = Math.max(sharpState.maxInFlight, sharpState.inFlight);
+        sharpState.renders.push(filePath);
+        // Yield, so an overlapping caller would be observed rather than serialized by luck.
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        sharpState.inFlight -= 1;
+        return Buffer.from([0x89, 0x50, 0x4e, 0x47]);
+      },
+    };
+    return chain;
+  };
+  const sharp = Object.assign(pipeline, {
+    concurrency: (value: number) => {
+      sharpState.concurrencyCalls.push(value);
+      return value;
+    },
+  });
+  return { default: sharp };
+});
+
 vi.mock("@google/genai", () => ({
   GoogleGenAI: class {
     models = { generateContent: generateContentMock };
+    constructor(options: Record<string, unknown>) {
+      clientOptions.push(options);
+    }
   },
 }));
 
@@ -414,3 +455,273 @@ describe("captionImagesWithGemini — Gemini provider", () => {
     });
   });
 });
+
+describe("captionImagesWithGemini — Vertex AI provider", () => {
+  const dirs: string[] = [];
+  const SERVICE_ACCOUNT = JSON.stringify({
+    type: "service_account",
+    project_id: "prefab-kit-000000",
+    private_key: "-----BEGIN PRIVATE KEY-----super-secret-material-----END PRIVATE KEY-----",
+    client_email: "capture@prefab-kit-000000.iam.gserviceaccount.com",
+  });
+
+  // Earlier describe blocks construct the SDK client too, so the record is cleared going in
+  // rather than only on the way out.
+  beforeEach(() => {
+    clientOptions.length = 0;
+  });
+
+  afterEach(() => {
+    generateContentMock.mockReset();
+    clientOptions.length = 0;
+    vi.unstubAllEnvs();
+    for (const d of dirs) rmSync(d, { recursive: true, force: true });
+    dirs.length = 0;
+  });
+
+  function vertexEnv(): void {
+    vi.stubEnv("OPENROUTER_API_KEY", "");
+    vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
+    vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", SERVICE_ACCOUNT);
+  }
+
+  it("prefers a service account over a bare API key, and authenticates against the project", async () => {
+    // A deployment can hold a Gemini key that is present but rejected. Every request then
+    // returns empty text, so the capture reports "Captioned N/N" followed by "0 images
+    // captioned" and no error — which is exactly what production was doing.
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vertexEnv();
+    vi.stubEnv("GEMINI_API_KEY", "a-key-that-the-server-would-be-rejected-for");
+    generateContentMock.mockResolvedValue({ text: "A dark blue product screenshot." });
+
+    const stages: string[] = [];
+    const captions = await captionImagesWithGemini(
+      dir,
+      (stage, detail) => {
+        stages.push(detail ?? stage);
+      },
+      [],
+    );
+
+    expect(captions).toEqual({ "hero.png": "A dark blue product screenshot." });
+    expect(stages.join(" ")).toContain("Vertex AI");
+    expect(clientOptions).toHaveLength(1);
+    expect(clientOptions[0]).toMatchObject({
+      vertexai: true,
+      project: "prefab-kit-000000",
+      location: "us-central1",
+      googleAuthOptions: { credentials: JSON.parse(SERVICE_ACCOUNT) },
+    });
+    expect(clientOptions[0]).not.toHaveProperty("apiKey");
+  });
+
+  it("honours an explicit region", async () => {
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vertexEnv();
+    vi.stubEnv("HYPERFRAMES_VERTEX_LOCATION", "europe-west4");
+    generateContentMock.mockResolvedValue({ text: "A caption." });
+
+    await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(clientOptions[0]).toMatchObject({ location: "europe-west4" });
+  });
+
+  it("spends no output budget on thinking", async () => {
+    // Thinking tokens come out of maxOutputTokens, so a model left free to think can consume
+    // the whole budget and return empty text: a successful request that produces no caption.
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vertexEnv();
+    // Capture the request inside the mock, where the argument is well-typed — avoids
+    // indexing `mock.calls` (and the repo's ban on `as` assertions).
+    let model: unknown;
+    let thinkingConfig: unknown;
+    generateContentMock.mockImplementation(
+      async (request: { model: string; config?: { thinkingConfig?: unknown } }) => {
+        model = request.model;
+        thinkingConfig = request.config?.thinkingConfig;
+        return { text: "A caption." };
+      },
+    );
+
+    await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(generateContentMock).toHaveBeenCalledTimes(1);
+    expect(thinkingConfig).toEqual({ thinkingBudget: 0 });
+    // The API's flash-lite preview id is not resolvable on Vertex, so the default differs.
+    expect(model).toBe("gemini-2.5-flash");
+  });
+
+  it("skips captioning when the service account is unparseable, without echoing it", async () => {
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vi.stubEnv("OPENROUTER_API_KEY", "");
+    vi.stubEnv("GEMINI_API_KEY", "");
+    vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
+    vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", "{not-json super-secret-material");
+
+    const warnings: string[] = [];
+    let outcome: VisionCaptionOutcome | undefined;
+    const captions = await captionImagesWithGemini(dir, () => {}, warnings, {
+      onOutcome: (value) => {
+        outcome = value;
+      },
+    });
+
+    expect(captions).toEqual({});
+    expect(generateContentMock).not.toHaveBeenCalled();
+    expect(warnings.join(" ")).toContain("not valid JSON");
+    expect(warnings.join(" ")).not.toContain("super-secret-material");
+    if (!outcome) throw new Error("Expected vision caption outcome");
+    expect(outcome.internalError).toBe(true);
+    expect(resolveVisionPhaseCompletion(outcome, 10_000)).toEqual({
+      status: "degraded",
+      reason: "internal-error",
+    });
+  });
+
+  it("stays on OpenRouter when the user has opted into it explicitly", async () => {
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vertexEnv();
+    vi.stubEnv("OPENROUTER_API_KEY", "or-key");
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      json: async () => ({ choices: [{ message: { content: "A caption." } }] }),
+    });
+    vi.stubGlobal("fetch", fetchMock);
+
+    await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(fetchMock).toHaveBeenCalled();
+    expect(clientOptions).toHaveLength(0);
+    vi.unstubAllGlobals();
+  });
+
+  it("needs both halves of the credential before it will use Vertex", async () => {
+    const dir = makeProjectWithImages();
+    dirs.push(dir);
+    vi.stubEnv("OPENROUTER_API_KEY", "");
+    vi.stubEnv("GEMINI_API_KEY", "");
+    vi.stubEnv("HYPERFRAMES_VERTEX_PROJECT_ID", "prefab-kit-000000");
+    vi.stubEnv("HYPERFRAMES_VERTEX_SERVICE_ACCOUNT", "");
+
+    const captions = await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(captions).toEqual({});
+    expect(clientOptions).toHaveLength(0);
+    expect(generateContentMock).not.toHaveBeenCalled();
+  });
+});
+
+describe("captionImagesWithGemini — SVG rasterization", () => {
+  const dirs: string[] = [];
+
+  beforeEach(() => {
+    sharpState.inFlight = 0;
+    sharpState.maxInFlight = 0;
+    sharpState.concurrencyCalls.length = 0;
+    sharpState.renders.length = 0;
+    clientOptions.length = 0;
+  });
+
+  afterEach(() => {
+    generateContentMock.mockReset();
+    vi.unstubAllGlobals();
+    vi.unstubAllEnvs();
+    for (const d of dirs) rmSync(d, { recursive: true, force: true });
+    dirs.length = 0;
+  });
+
+  function makeProjectWithSvgs(count: number): string {
+    const dir = mkdtempSync(join(tmpdir(), "hf-svg-"));
+    mkdirSync(join(dir, "assets", "svgs"), { recursive: true });
+    for (let i = 0; i < count; i++) {
+      writeFileSync(
+        join(dir, "assets", "svgs", `logo-${i}.svg`),
+        '<svg xmlns="http://www.w3.org/2000/svg"><path fill="#000" d="M0 0h8v8H0z"/></svg>',
+      );
+    }
+    return dir;
+  }
+
+  it("never runs two libvips renders at once", async () => {
+    // Production aborted here with `free(): unaligned chunk detected in tcache 2` (SIGABRT),
+    // twice in fourteen days, losing the whole capture. A native abort cannot be caught by the
+    // surrounding try/catch, so the concurrency has to be absent rather than handled — which
+    // makes "one at a time" the assertion, not "errors are reported".
+    const dir = makeProjectWithSvgs(6);
+    dirs.push(dir);
+    vi.stubEnv("OPENROUTER_API_KEY", "or-key");
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => {
+        // Slower than a render, so overlapping renders would be the easy way to go faster —
+        // the point is that the loop does not take it.
+        await new Promise((resolve) => setTimeout(resolve, 15));
+        return new Response(
+          JSON.stringify({ choices: [{ message: { content: "A dark glyph." } }] }),
+          {
+            status: 200,
+            headers: { "content-type": "application/json" },
+          },
+        );
+      }),
+    );
+
+    const captions = await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(sharpState.renders).toHaveLength(6);
+    expect(sharpState.maxInFlight).toBe(1);
+    expect(Object.keys(captions)).toHaveLength(6);
+  });
+
+  it("bounds libvips' own worker pool as well", async () => {
+    // Left at its default the pool sizes itself to the host's core count, so serializing the
+    // loop alone still leaves one render fanning out across every core.
+    const dir = makeProjectWithSvgs(1);
+    dirs.push(dir);
+    vi.stubEnv("OPENROUTER_API_KEY", "or-key");
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(JSON.stringify({ choices: [{ message: { content: "A glyph." } }] }), {
+            status: 200,
+            headers: { "content-type": "application/json" },
+          }),
+      ),
+    );
+
+    await captionImagesWithGemini(dir, () => {}, []);
+
+    expect(sharpState.concurrencyCalls).toEqual([1]);
+  });
+
+  it("keeps captioning the rest when one SVG cannot be rasterized", async () => {
+    const dir = makeProjectWithSvgs(3);
+    dirs.push(dir);
+    vi.stubEnv("OPENROUTER_API_KEY", "or-key");
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(
+        async () =>
+          new Response(JSON.stringify({ choices: [{ message: { content: "A glyph." } }] }), {
+            status: 200,
+            headers: { "content-type": "application/json" },
+          }),
+      ),
+    );
+    // One file is unreadable, which is how an exotic SVG behaves through sharp.
+    rmSync(join(dir, "assets", "svgs", "logo-1.svg"));
+    mkdirSync(join(dir, "assets", "svgs", "logo-1.svg"));
+
+    const warnings: string[] = [];
+    const captions = await captionImagesWithGemini(dir, () => {}, warnings);
+
+    expect(Object.keys(captions).sort()).toEqual(["svgs/logo-0.svg", "svgs/logo-2.svg"]);
+    expect(sharpState.maxInFlight).toBe(1);
+  });
+});