[POC] Remove background / Mask people

Metal/PersonMask.metal

@@ -0,0 +1,11 @@
+#include <CoreImage/CoreImage.h>
+using namespace metal;
+
+// Cuts the baked matte (coverage in its red channel) into source's alpha; output must be
+// premultiplied ourselves or masked-out regions bleed color through when composited.
+extern "C" float4 personMask(coreimage::sample_t source, coreimage::sample_t matte, float invert) {
+    float coverage = matte.r;
+    float a = invert > 0.5 ? 1.0 - coverage : coverage;
+    float outAlpha = source.a * a;
+    return float4(source.rgb * outAlpha, outAlpha);
+}

Sources/PalmierPro/Agent/Tools/ToolDefinitions.swift

@@ -33,6 +33,7 @@ enum ToolName: String, CaseIterable, Sendable {
     case searchMedia = "search_media"
     case applyColor = "apply_color"
     case applyEffect = "apply_effect"
+    case removeBackground = "remove_background"
     case inspectColor = "inspect_color"
     case listFolders = "list_folders"
     case createFolder = "create_folder"
@@ -761,6 +762,16 @@ enum ToolDefinitions {
                 required: ["clipIds"]
             )
         ),
+        AgentTool(
+            name: .removeBackground,
+            description: "Removes the background from video clips using on-device person detection: finds every person in each clip and bakes a matte that keeps them, cutting everything else — the same pipeline as the Inspector's 'Remove Background' button. Applies as an editable key.personMask effect (adjust after with apply_effect's invert/feather params, or remove: ['key.personMask']). Baking runs per clip and can take a while for longer clips; this call blocks until it finishes (or fails, e.g. no people detected) and reports a per-clip result — no polling needed. Undoable.",
+            inputSchema: objectSchema(
+                properties: [
+                    "clipIds": ["type": "array", "items": ["type": "string"], "description": "Video clip ids from get_timeline."],
+                ],
+                required: ["clipIds"]
+            )
+        ),
         AgentTool(
             name: .applyColor,
             description: "Author/refine a color grade on video/image clips with named controls — the colorist path, distinct from apply_effect (looks/FX). MERGES with the clip's current grade: only the params you pass change, the rest are preserved, so you can nudge one knob at a time (pass reset:true to start from neutral). Applies as live, editable color.* effects; non-color effects untouched. Iterate: apply_color → inspect_color(clipId, reference) → read the gap → adjust → repeat. Undoable. All knobs optional. Color WHEELS use HUE (0–360°, standard) + AMOUNT per tonal zone — to push shadows teal, set shadowsHue 180 and shadowsAmount ~0.15. CURVES (master + per-channel R/G/B) give precise tone shaping — per-channel curves are tone-selective (e.g. pull the blue curve down in the highlights to tame a bright sky). HUE CURVES do secondary/qualified correction — target a source hue and shift its hue/saturation/lightness (e.g. desaturate greens, warm the skin) without a mask; pair with inspect_color's hueHistogram to find which hues are present. LUT applies a .cube film-look pack on top of the grade.",

Sources/PalmierPro/Agent/Tools/ToolExecutor+PersonMask.swift

@@ -0,0 +1,41 @@
+import Foundation
+
+extension ToolExecutor {
+    fileprivate struct RemoveBackgroundInput: DecodableToolArgs {
+        let clipIds: [String]
+        static let allowedKeys: Set<String> = ["clipIds"]
+    }
+
+    /// Bakes a per-person matte for each clip, joining an already-running bake if one exists.
+    func removeBackground(_ editor: EditorViewModel, _ args: [String: Any]) async throws -> ToolResult {
+        let input: RemoveBackgroundInput = try decodeToolArgs(args, path: "remove_background")
+        guard !input.clipIds.isEmpty else { throw ToolError("clipIds is empty.") }
+        for id in input.clipIds {
+            guard let clip = editor.clipFor(id: id) else { throw ToolError("Clip not found: \(id)") }
+            guard clip.mediaType == .video else {
+                throw ToolError("Clip \(id) is a \(clip.mediaType.rawValue) clip; remove_background needs a video clip.")
+            }
+        }
+
+        var succeeded: [String] = []
+        var failed: [(id: String, message: String)] = []
+        for id in input.clipIds {
+            do {
+                try await editor.removeBackground(clipId: id).value
+                succeeded.append(id)
+            } catch {
+                failed.append((id, error.localizedDescription))
+            }
+        }
+
+        guard !succeeded.isEmpty else {
+            let detail = failed.map { "\($0.id): \($0.message)" }.joined(separator: "; ")
+            throw ToolError("Background removal failed on all \(failed.count) clip(s): \(detail)")
+        }
+        var summary = "Removed background on \(succeeded.count) clip\(succeeded.count == 1 ? "" : "s"): \(succeeded.joined(separator: ", "))."
+        if !failed.isEmpty {
+            summary += " Failed on \(failed.count): " + failed.map { "\($0.id) (\($0.message))" }.joined(separator: "; ") + "."
+        }
+        return .ok(summary + " Verify with inspect_timeline.")
+    }
+}

Sources/PalmierPro/Agent/Tools/ToolExecutor.swift

@@ -81,6 +81,7 @@ final class ToolExecutor {
         case .searchMedia:   return try await searchMedia(editor, args)
         case .applyColor:    return try applyColor(editor, args)
         case .applyEffect:   return try applyEffect(editor, args)
+        case .removeBackground: return try await removeBackground(editor, args)
         case .inspectColor:  return try await inspectColor(editor, args)
         case .addClips:         return try addClips(editor, args)
         case .insertClips:      return try insertClips(editor, args)

Sources/PalmierPro/Compositing/CompositorInstruction.swift

@@ -10,6 +10,8 @@ struct LayerPlan: Sendable {
     let clip: Clip
     let natSize: CGSize
     let preferredTransform: CGAffineTransform
+    /// Composition track carrying this clip's baked person-mask matte, if any — see `PersonMaskBaker`.
+    var personMaskTrackID: CMPersistentTrackID? = nil
 
     var trackID: CMPersistentTrackID? {
         if case .track(let id) = source { return id }
@@ -35,10 +37,13 @@ final class CompositorInstruction: NSObject, AVVideoCompositionInstructionProtoc
         self.renderSize = renderSize
         self.fps = fps
         var seen = Set<CMPersistentTrackID>()
-        self.requiredSourceTrackIDs = layers.compactMap {
-            guard let id = $0.trackID else { return nil }  // text layers need no decoded source
-            return seen.insert(id).inserted ? NSNumber(value: id) : nil
+        var ids: [NSValue] = []
+        for layer in layers {
+            for id in [layer.trackID, layer.personMaskTrackID].compactMap({ $0 }) where seen.insert(id).inserted {
+                ids.append(NSNumber(value: id))
+            }
         }
+        self.requiredSourceTrackIDs = ids
         super.init()
     }
 }

Sources/PalmierPro/Compositing/EffectRegistry.swift

@@ -338,6 +338,16 @@ enum EffectRegistry {
                                       softness: p.value("softness"), spill: p.value("spill"))
             }
         ),
+        // apply is a no-op: FrameRenderer special-cases this id to use the baked matte track instead.
+        EffectDescriptor(
+            id: "key.personMask", displayName: "People Mask", category: "Key",
+            params: [
+                EffectParamSpec(key: "invert", label: "Invert", range: 0...1, defaultValue: 0, unit: ""),
+                EffectParamSpec(key: "feather", label: "Feather", range: 0...1, defaultValue: 0, unit: ""),
+            ],
+            resourceKey: "maskCachePath",
+            apply: { image, _, _ in image }
+        ),
     ]
 
     static let byId: [String: EffectDescriptor] = Dictionary(
@@ -350,7 +360,7 @@ enum EffectRegistry {
     static let canonicalOrder: [String] = [
         "color.exposure", "color.contrast", "color.highlightsShadows", "color.blacksWhites",
         "color.temperature", "color.vibrance", "color.saturation", "color.wheels", "color.curves",
-        "color.hueCurves", "color.lut", "detail.clarity", "key.chroma", "blur.gaussian", "blur.sharpen",
+        "color.hueCurves", "color.lut", "detail.clarity", "key.chroma", "key.personMask", "blur.gaussian", "blur.sharpen",
         "blur.noiseReduction", "blur.motion", "stylize.grain", "stylize.vignette", "stylize.glow",
     ]
 

Sources/PalmierPro/Compositing/FrameRenderer.swift

@@ -25,7 +25,8 @@ enum FrameRenderer {
             switch layer.source {
             case .track(let id):
                 guard let buffer = sourceFrame(id) else { continue }
-                image = composedLayer(layer, buffer: buffer, frame: frame,
+                let matteBuffer = layer.personMaskTrackID.flatMap { sourceFrame($0) }
+                image = composedLayer(layer, buffer: buffer, matteBuffer: matteBuffer, frame: frame,
                                       renderSize: instruction.renderSize, bakeOpacity: isNormal)
             case .text:
                 image = composedTextLayer(layer, frame: frame, renderSize: instruction.renderSize,
@@ -69,6 +70,7 @@ enum FrameRenderer {
     private static func composedLayer(
         _ layer: LayerPlan,
         buffer: CVPixelBuffer,
+        matteBuffer: CVPixelBuffer? = nil,
         frame: Int,
         renderSize: CGSize,
         bakeOpacity: Bool = true
@@ -83,6 +85,11 @@ enum FrameRenderer {
             .unpremultiplyingAlpha()
         let srcHeight = CGFloat(CVPixelBufferGetHeight(buffer))
 
+        // The person-mask matte is baked at the same size as the main track and stays
+        // frame-locked to it (see CompositionBuilder), so the same crop rect applies to both.
+        var matte = matteBuffer.map { CIImage(cvPixelBuffer: $0, options: [.colorSpace: NSNull()]) }
+        let matteSrcHeight = matteBuffer.map { CGFloat(CVPixelBufferGetHeight($0)) } ?? srcHeight
+
         let crop = clip.cropAt(frame: frame)
         if !crop.isIdentity {
             // Display-space insets → source pixels → CI's bottom-left origin.
@@ -98,12 +105,22 @@ enum FrameRenderer {
                 width: avRect.width,
                 height: avRect.height
             ))
+            matte = matte?.cropped(to: CGRect(
+                x: avRect.origin.x,
+                y: matteSrcHeight - avRect.origin.y - avRect.height,
+                width: avRect.width,
+                height: avRect.height
+            ))
         }
 
         // Effects apply in source-pixel space: after crop, before placement.
         if let effects = clip.effects, !effects.isEmpty {
             let offset = frame - clip.startFrame
             for effect in effects where effect.enabled {
+                if effect.type == "key.personMask" {
+                    image = PersonMaskKernel.apply(image, matte: matte, effect: effect, atOffset: offset)
+                    continue
+                }
                 guard let descriptor = EffectRegistry.descriptor(id: effect.type) else { continue }
                 image = descriptor.render(image, effect: effect, atOffset: offset)
             }

Sources/PalmierPro/Compositing/Kernels/PersonMaskKernel.swift

@@ -0,0 +1,29 @@
+import CoreImage
+import Foundation
+
+/// Cuts a baked person-mask matte into a clip's alpha via a Metal kernel. Kernel: `Metal/PersonMask.metal`.
+enum PersonMaskKernel {
+    private static let kernel = CIKernelLoader.colorKernel("PersonMask", "personMask")
+    static let maxFeatherRadius: Double = 40
+
+    /// Fails open (returns `image` unchanged) when there's no kernel or no matte track —
+    /// e.g. the bake cache was cleared. Feathering softens the matte edge before the cut.
+    static func apply(_ image: CIImage, matte: CIImage?, effect: Effect, atOffset offset: Int) -> CIImage {
+        guard let kernel, let matte else { return image }
+        let invert = (effect.params["invert"]?.resolved(at: offset, default: 0) ?? 0) >= 0.5
+        let feather = effect.params["feather"]?.resolved(at: offset, default: 0) ?? 0
+
+        var softened = matte
+        if feather > 0 {
+            let radius = feather * Self.maxFeatherRadius
+            softened = softened
+                .applyingFilter("CIGaussianBlur", parameters: [kCIInputRadiusKey: radius])
+                .cropped(to: matte.extent)
+        }
+
+        return kernel.apply(
+            extent: image.extent,
+            arguments: [image, softened, invert ? Float(1) : Float(0)]
+        ) ?? image
+    }
+}

Sources/PalmierPro/Compositing/PersonMask/PersonMaskAnalyzer.swift

@@ -0,0 +1,78 @@
+import AVFoundation
+import CoreImage
+import Vision
+
+/// Detects people in a clip's first frame; the bake pass then tracks forward from there.
+enum PersonMaskAnalyzer {
+    struct Candidate: Identifiable, Sendable {
+        let id: Int   // Vision instance label — only meaningful within one seed-frame observation
+        let thumbnail: CGImage
+    }
+
+    enum AnalyzerError: LocalizedError, Equatable {
+        case noFrame
+        case noPeople
+
+        var errorDescription: String? {
+            switch self {
+            case .noFrame: "Could not read a frame to analyze."
+            case .noPeople: "No people detected in this clip's first frame."
+            }
+        }
+    }
+
+    private static let thumbnailContext = CIContext(options: [.workingColorSpace: NSNull()])
+    private static let thumbnailMaxDimension: CGFloat = 160
+
+    /// Detects people in the first frame of `url`'s video track, in raw sensor orientation to
+    /// match `PersonMaskBaker`'s per-frame Vision passes regardless of rotation metadata.
+    static func detectPeople(url: URL) async throws -> [Candidate] {
+        guard let frame = await frameImage(url: url) else { throw AnalyzerError.noFrame }
+        return try await detectPeople(in: frame)
+    }
+
+    static func detectPeople(in image: CIImage) async throws -> [Candidate] {
+        guard let observation = try await observation(in: image) else { throw AnalyzerError.noPeople }
+        var candidates: [Candidate] = []
+        for label in observation.allInstances.sorted() {
+            guard let maskBuffer = try? observation.generateMask(for: IndexSet(integer: label)) else { continue }
+            let mask = CIImage(cvPixelBuffer: maskBuffer)
+            guard let thumbnail = thumbnail(of: image, mask: mask) else { continue }
+            candidates.append(Candidate(id: label, thumbnail: thumbnail))
+        }
+        guard !candidates.isEmpty else { throw AnalyzerError.noPeople }
+        return candidates
+    }
+
+    static func observation(in image: CIImage) async throws -> InstanceMaskObservation? {
+        let request = GeneratePersonInstanceMaskRequest()
+        return try await request.perform(on: image)
+    }
+
+    private static func thumbnail(of image: CIImage, mask: CIImage) -> CGImage? {
+        let extent = image.extent
+        guard extent.width > 0, extent.height > 0, extent.width.isFinite, extent.height.isFinite else { return nil }
+        let maskExtent = mask.extent
+        let scaledMask = mask.transformed(by: CGAffineTransform(
+            scaleX: extent.width / max(1, maskExtent.width),
+            y: extent.height / max(1, maskExtent.height)
+        ))
+        let cutout = image.applyingFilter("CIBlendWithMask", parameters: [
+            kCIInputBackgroundImageKey: CIImage(color: .clear).cropped(to: extent),
+            kCIInputMaskImageKey: scaledMask,
+        ])
+        let scale = min(1, thumbnailMaxDimension / max(extent.width, extent.height))
+        let scaled = cutout.transformed(by: CGAffineTransform(scaleX: scale, y: scale))
+        return thumbnailContext.createCGImage(scaled, from: scaled.extent)
+    }
+
+    private static func frameImage(url: URL) async -> CIImage? {
+        let generator = AVAssetImageGenerator(asset: AVURLAsset(url: url))
+        generator.appliesPreferredTrackTransform = false // raw orientation — see detectPeople(url:)
+        let tolerance = CMTime(seconds: 1, preferredTimescale: 600)
+        generator.requestedTimeToleranceBefore = tolerance
+        generator.requestedTimeToleranceAfter = tolerance
+        guard let cg = try? await generator.image(at: .zero).image else { return nil }
+        return CIImage(cgImage: cg, options: [.colorSpace: NSNull()])
+    }
+}