Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions contributors.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
44 changes: 26 additions & 18 deletions packages/@ant/computer-use-swift/src/backends/darwin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,25 +159,33 @@ export const apps: AppsAPI = {

async listInstalled() {
try {
const result = await osascript(`
tell application "System Events"
set appList to ""
repeat with appFile in (every file of folder "Applications" of startup disk whose name ends with ".app")
set appPath to POSIX path of (appFile as alias)
set appName to name of appFile
set appList to appList & appPath & "|" & appName & "\\n"
end repeat
return appList
end tell
`)
return result.split('\n').filter(Boolean).map(line => {
const [path, name] = line.split('|', 2)
const displayName = (name ?? '').replace(/\.app$/, '')
return {
bundleId: `com.app.${displayName.toLowerCase().replace(/\s+/g, '-')}`,
displayName,
path: path ?? '',
// Use mdls to enumerate apps and get real bundle identifiers.
// The previous AppleScript approach generated fake bundle IDs
// (com.app.display-name) which prevented request_access from matching
// apps by their real bundle ID (e.g. com.google.Chrome).
const dirs = ['/Applications', '~/Applications', '/System/Applications']
const allApps: InstalledApp[] = []
for (const dir of dirs) {
const expanded = dir.startsWith('~') ? join(process.env.HOME ?? '~', dir.slice(1)) : dir
const proc = Bun.spawn(
['bash', '-c', `for f in "${expanded}"/*.app; do [ -d "$f" ] || continue; bid=$(mdls -name kMDItemCFBundleIdentifier "$f" 2>/dev/null | sed 's/.*= "//;s/"//'); name=$(basename "$f" .app); echo "$f|$name|$bid"; done`],
{ stdout: 'pipe', stderr: 'pipe' },
Comment on lines +166 to +172
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Scan nested app folders too.

Line 171 only matches *.app directly under each root. That misses common locations like /System/Applications/Utilities/Terminal.app and vendor subfolders under /Applications, so those apps never reach listInstalled() and cannot be granted/launched later.

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@packages/`@ant/computer-use-swift/src/backends/darwin.ts around lines 166 -
172, The current scan only matches "*.app" at the top level and misses nested
app bundles; update the command built for Bun.spawn (where proc is created using
expanded from dirs) to recursively find .app bundles (e.g., use find
"${expanded}" -type d -name '*.app' -print0 and iterate over results) so nested
apps like /System/Applications/Utilities/Terminal.app are included; adjust the
parsing logic that reads each result (the code that extracts path, name, and
bundle id) to handle the find output (null-separated or newline-separated) and
continue populating allApps/InstalledApp entries accordingly.

)
const text = await new Response(proc.stdout).text()
await proc.exited
for (const line of text.split('\n').filter(Boolean)) {
const [path, displayName, bundleId] = line.split('|', 3)
if (path && displayName && bundleId && bundleId !== '(null)') {
allApps.push({ bundleId, displayName, path })
Comment on lines +171 to +179
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Don't use the bundle directory name as displayName.

resolveRequestedApps() in packages/@ant/computer-use-mcp/src/toolCalls.ts, Lines 791-830 matches user requests against InstalledApp.displayName. Using basename "$f" here changes that field to the filesystem name, so localized/system apps can stop matching what the user actually sees. Populate displayName from bundle metadata and only fall back to the basename when metadata is missing.

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@packages/`@ant/computer-use-swift/src/backends/darwin.ts around lines 171 -
179, The current loop sets displayName from the bundle directory name (basename
"$f"), which breaks matching against InstalledApp.displayName; update the shell
pipeline in the for loop that builds each line so it extracts a localized
display name from bundle metadata (e.g., use mdls -name kMDItemDisplayName or
fall back to kMDItemCFBundleName/kMDItemDisplayName extraction) and only use
basename "$f" when no metadata display name is present, then keep using the
parsed displayName variable when pushing into allApps (refer to the for (...)
command that defines path|displayName|bundleId and the allApps.push({ bundleId,
displayName, path }) code).

}
}
}
// Deduplicate by bundleId (prefer /Applications over ~/Applications)
const seen = new Set<string>()
return allApps.filter(app => {
if (seen.has(app.bundleId)) return false
seen.add(app.bundleId)
return true
})
} catch {
return []
Expand Down
6 changes: 3 additions & 3 deletions src/utils/computerUse/escHotkey.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export function registerEscHotkey(onEscape: () => void): boolean {
if (process.platform !== 'darwin') return false
if (registered) return true
const cu = requireComputerUseSwift()
if (!(cu as any).hotkey.registerEscape(onEscape)) {
if (!(cu as any).hotkey?.registerEscape(onEscape)) {
// CGEvent.tapCreate failed — typically missing Accessibility permission.
// CU still works, just without ESC abort. Mirrors Cowork's escAbort.ts:81.
logForDebugging('[cu-esc] registerEscape returned false', { level: 'warn' })
Expand All @@ -41,7 +41,7 @@ export function registerEscHotkey(onEscape: () => void): boolean {
export function unregisterEscHotkey(): void {
if (!registered) return
try {
(requireComputerUseSwift() as any).hotkey.unregister()
(requireComputerUseSwift() as any).hotkey?.unregister()
} finally {
releasePump()
registered = false
Expand All @@ -51,5 +51,5 @@ export function unregisterEscHotkey(): void {

export function notifyExpectedEscape(): void {
if (!registered) return
(requireComputerUseSwift() as any).hotkey.notifyExpectedEscape()
(requireComputerUseSwift() as any).hotkey?.notifyExpectedEscape()
}
47 changes: 45 additions & 2 deletions src/utils/computerUse/hostAdapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,38 @@ class DebugLogger implements Logger {
}
}

// ---------------------------------------------------------------------------
// JXA-based TCC permission probes (fallback when native .node module absent)
// ---------------------------------------------------------------------------

/** Probe accessibility by asking System Events for a process list. */
function checkAccessibilityJXA(): boolean {
try {
const result = Bun.spawnSync({
cmd: ['osascript', '-e', 'tell application "System Events" to get name of every process whose background only is false'],
stdout: 'pipe',
stderr: 'pipe',
})
return result.exitCode === 0
} catch {
return false
}
}

/** Probe screen recording by attempting a 1x1 screencapture. */
function checkScreenRecordingJXA(): boolean {
try {
const result = Bun.spawnSync({
cmd: ['screencapture', '-x', '-R', '0,0,1,1', '/dev/null'],
stdout: 'pipe',
stderr: 'pipe',
})
return result.exitCode === 0
} catch {
return false
}
}

let cached: ComputerUseHostAdapter | undefined

/**
Expand All @@ -47,8 +79,19 @@ export function getComputerUseHostAdapter(): ComputerUseHostAdapter {
ensureOsPermissions: async () => {
if (process.platform !== 'darwin') return { granted: true }
const cu = requireComputerUseSwift()
const accessibility = (cu as any).tcc.checkAccessibility()
const screenRecording = (cu as any).tcc.checkScreenRecording()
const tcc = (cu as any).tcc
// Native Swift .node module provides tcc.checkAccessibility/checkScreenRecording.
// When absent (decompiled/reverse-engineered build), fall back to JXA probes.
if (tcc) {
const accessibility = tcc.checkAccessibility()
const screenRecording = tcc.checkScreenRecording()
return accessibility && screenRecording
? { granted: true }
: { granted: false, accessibility, screenRecording }
}
// JXA fallback: try to query System Events (accessibility) and screencapture (screen recording).
const accessibility = checkAccessibilityJXA()
const screenRecording = checkScreenRecordingJXA()
return accessibility && screenRecording
? { granted: true }
: { granted: false, accessibility, screenRecording }
Expand Down