fix(core): tell HEIC, QuickTime and M4A apart from MP4 (#58887)

Author: rekram1-node

packages/core/src/mime.ts

@@ -8,11 +8,26 @@ export function detect(bytes: Uint8Array) {
   if (startsWith(bytes, [0x25, 0x50, 0x44, 0x46, 0x2d])) return "application/pdf"
   if (startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes.subarray(8), [0x57, 0x45, 0x42, 0x50]))
     return "image/webp"
-  if (
-    startsWith(bytes.subarray(4), [0x66, 0x74, 0x79, 0x70]) &&
-    (startsWith(bytes.subarray(8), [0x61, 0x76, 0x69, 0x66]) || startsWith(bytes.subarray(8), [0x61, 0x76, 0x69, 0x73]))
-  )
-    return "image/avif"
+  if (bytes.length >= 12 && ascii(bytes, 4, 8) === "ftyp") {
+    // AVIF, HEIC, QuickTime and M4A share MP4's container; the major brand tells them apart, and generic HEIF brands
+    // list `avif`/`avis` among the compatible brands that follow the minor version.
+    const brand = ascii(bytes, 8, 12)
+    const boxEnd = Math.min(new DataView(bytes.buffer, bytes.byteOffset).getUint32(0), bytes.length)
+    const compatible = Array.from({ length: Math.floor((boxEnd - 16) / 4) }, (_, index) =>
+      ascii(bytes, 16 + index * 4, 20 + index * 4),
+    )
+    const generic = brand === "mif1" || brand === "msf1"
+    if (
+      brand === "avif" ||
+      brand === "avis" ||
+      (generic && (compatible.includes("avif") || compatible.includes("avis")))
+    )
+      return "image/avif"
+    if (generic || ["heic", "heix", "hevc", "hevx"].includes(brand)) return "image/heic"
+    if (brand === "qt  ") return "video/quicktime"
+    if (brand === "M4A ") return "audio/mp4"
+    return "video/mp4"
+  }
   return isText(bytes) ? "text/plain" : "application/octet-stream"
 }
 
@@ -26,6 +41,10 @@ export const isTextual = (mime: string) =>
   mime === "application/javascript" ||
   mime === "application/x-javascript"
 
+function ascii(bytes: Uint8Array, start: number, end: number) {
+  return String.fromCharCode(...bytes.subarray(start, end))
+}
+
 function startsWith(bytes: Uint8Array, prefix: number[]) {
   return prefix.every((value, index) => bytes[index] === value)
 }

packages/core/src/session/prompt.ts

@@ -19,6 +19,9 @@ import { SessionBlob } from "./blob.js"
 import { AttachmentError, SkillNotFoundError } from "./error.js"
 import { SessionStored } from "./stored.js"
 
+// Image types the resizer can decode; others such as HEIC and AVIF go to the provider unchanged.
+const RESIZABLE_IMAGE_MIMES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp", "image/bmp"])
+
 export type Input = {
   text: string
   files?: PromptInput.Prompt["files"]
@@ -146,7 +149,7 @@ const normalizeImageAttachment = Effect.fn("SessionPrompt.normalizeImageAttachme
   data: string,
   mime: string,
 ) {
-  if (!mime.startsWith("image/")) return { data: Base64.make(data), mime }
+  if (!RESIZABLE_IMAGE_MIMES.has(mime)) return { data: Base64.make(data), mime }
   const image = yield* Image.Service
   const label = attachmentLabel(input)
   const content = { uri: label, content: data, encoding: "base64" as const, mime }

packages/core/test/mime.test.ts

@@ -0,0 +1,17 @@
+import { describe, expect, test } from "bun:test"
+import { Mime } from "@opencode/core/mime"
+
+const ftyp = (hex: string) => Uint8Array.from(Buffer.from(hex, "hex"))
+
+describe("Mime.detect", () => {
+  test("tells ISO-BMFF brands apart", () => {
+    expect(Mime.detect(ftyp("000000206674797061766966000000004d695072617669666d6961666d696631"))).toBe("image/avif")
+    expect(Mime.detect(ftyp("000000186674797061766973000000006d73663161766973"))).toBe("image/avif")
+    expect(Mime.detect(ftyp("00000018667479706d69663100000000617669666d69663100000000"))).toBe("image/avif")
+    expect(Mime.detect(ftyp("000000246674797068656963000000006d6966314d6950726d6961664d694842"))).toBe("image/heic")
+    expect(Mime.detect(ftyp("00000018667479706d69663100000000686569636d69663100000000"))).toBe("image/heic")
+    expect(Mime.detect(ftyp("000000206674797069736f6d0000020069736f6d69736f32617663316d703431"))).toBe("video/mp4")
+    expect(Mime.detect(ftyp("00000014667479707174202000000200717420200000000877696465"))).toBe("video/quicktime")
+    expect(Mime.detect(ftyp("0000001c667479704d344120000002004d34412069736f6d69736f3200000008"))).toBe("audio/mp4")
+  })
+})

packages/core/test/session-prompt.test.ts

@@ -94,8 +94,10 @@ const locations = makeGlobalNode({
               replacements: [Bus.node.replace(Layer.succeed(Bus.Service, bus))],
             }),
             Layer.mock(Image.Service, {
-              normalize: (_resource, content) =>
-                Effect.succeed(content.content.length > 5 * 1024 * 1024 ? { ...content, content: "AA==" } : content),
+              normalize: (resource, content) =>
+                content.mime === "image/heic"
+                  ? Effect.fail(new Image.DecodeError({ resource }))
+                  : Effect.succeed(content.content.length > 5 * 1024 * 1024 ? { ...content, content: "AA==" } : content),
             }),
             Layer.mock(Snapshot.Service, {
               capture: () => Effect.undefined,
@@ -520,6 +522,23 @@ describe("Session.prompt", () => {
     }),
   )
 
+  it.effect("passes images the resizer cannot decode through unchanged", () =>
+    Effect.gen(function* () {
+      yield* setup
+      const session = yield* Session.Service
+      const data = Buffer.from("000000186674797068656963000000006d69663168656963", "hex").toString("base64")
+
+      const message = yield* session.prompt({
+        sessionID,
+        text: "Inspect this photo",
+        files: [{ uri: `data:image/heic;base64,${data}` }],
+        resume: false,
+      })
+
+      expect(message.payload.files).toEqual([{ data, mime: "image/heic", source: { type: "inline" } }])
+    }),
+  )
+
   it.effect("sniffs data URL content instead of trusting its declared MIME", () =>
     Effect.gen(function* () {
       yield* setup