diff --git a/CLAUDE.md b/CLAUDE.md index bed1ea1..ba168dc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,28 +34,41 @@ swift package update ## Architecture Overview -This project implements an MCP server that provides GUI automation capabilities for macOS through 4 tools: +This project implements an MCP server that provides GUI automation capabilities for macOS through these tools: - `moveMouse`: Move cursor to x,y coordinates - `mouseClick`: Perform mouse clicks (left/right) - `scroll`: Scroll in any direction - `sendKeys`: Send keyboard shortcuts +- `getScreenSize`: Get the display dimensions +- `getPixelColor`: Get color of a pixel at x,y +- `captureScreen`: Capture full screen as base64 PNG +- `captureRegion`: Capture a region as base64 PNG +- `saveScreenshot`: Save screenshot to file ### Key Components -- **main.swift**: MCP server initialization and tool registration. All tool implementations are in this file. -- **Server+Extension.swift**: Extends MCP Server with `waitForDisconnection()` to keep the server running. +- **main.swift**: MCP server initialization and startup +- **Server+Extension.swift**: Extends MCP Server with `waitForDisconnection()` to keep the server running +- **ToolRegistry.swift**: Manages tool registration and execution +- **Tools/**: + - **Mouse/**: Mouse control tools (moveMouse, mouseClick) + - **Screen/**: Screen capture and scroll tools + - **Keyboard/**: Keyboard input tools (sendKeys) ### Dependencies - **mcp-swift-sdk**: MCP protocol implementation -- **SwiftAutoGUI** (v0.3.2): macOS automation library + - Repository: https://github.com/modelcontextprotocol/swift-sdk +- **SwiftAutoGUI** (v0.10.0+): macOS automation library + - Repository: https://github.com/NakaokaRei/SwiftAutoGUI + - Documentation: https://nakaokarei.github.io/SwiftAutoGUI/documentation/swiftautogui/ ## Important Notes - **Platform Requirements**: macOS 15.0+, Swift 6.0+ - **Security**: Requires full accessibility permissions in System Preferences - **Communication**: Uses stdio transport (stdin/stdout) -- **No test suite**: Currently no tests are implemented +- **Test suite**: Unit tests for tools are in Tests/swift-mcp-guiTests/ ## Development Workflow diff --git a/Package.resolved b/Package.resolved index 37fdb28..43ca9aa 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "55091f4b3f48c15481aae1b8d36711f476dcd560123bf0c4607352c4d70f4000", + "originHash" : "563bfb558ed9a5796a0fc0c7e16b4e3fbdd257fb93f9798aae49083da222232b", "pins" : [ { "identity" : "eventsource", @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/yeatse/opencv-spm.git", "state" : { - "revision" : "9b5943c3095350dedfb2bebd276f05643b7045d1", - "version" : "4.11.0" + "revision" : "141d91be1f53990b4f5d63e1373afba539a7914d", + "version" : "4.12.0" } }, { @@ -24,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "3d8596ed08bd13520157f0355e35caed215ffbfa", - "version" : "1.6.3" + "revision" : "ce592ae52f982c847a4efc0dd881cc9eb32d29f2", + "version" : "1.6.4" } }, { @@ -42,8 +42,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system.git", "state" : { - "revision" : "61e4ca4b81b9e09e2ec863b00c340eb13497dac6", - "version" : "1.5.0" + "revision" : "41daa93a5d229e1548ec86ab527ce4783ca84dda", + "version" : "1.6.0" } }, { @@ -51,8 +51,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/NakaokaRei/SwiftAutoGUI.git", "state" : { - "revision" : "fbd5fb783b017b6312d3d370943759a2d37626e2", - "version" : "0.6.0" + "revision" : "8c046eea5788e9bbb84091b0b0223c9dbd980f44", + "version" : "0.11.0" } } ], diff --git a/Package.swift b/Package.swift index 2cb55a8..d932618 100644 --- a/Package.swift +++ b/Package.swift @@ -10,7 +10,7 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/modelcontextprotocol/swift-sdk.git", from: "0.9.0"), - .package(url: "https://github.com/NakaokaRei/SwiftAutoGUI.git", from: "0.5.0") + .package(url: "https://github.com/NakaokaRei/SwiftAutoGUI.git", from: "0.10.0") ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. diff --git a/README.md b/README.md index d694010..38562a5 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,22 @@ The server provides the following tools for controlling macOS: - File format is determined by the filename extension (.jpg, .jpeg, .png) - Quality parameter only affects JPEG files +### 10. Execute AppleScript +- Tool name: `executeAppleScript` +- Input: + - `script`: string (AppleScript code to execute) +- Executes AppleScript code directly and returns the result +- Returns "AppleScript Result: " if the script returns a value +- Returns "AppleScript executed successfully (no result returned)" if the script completes without returning a value + +### 11. Execute AppleScript File +- Tool name: `executeAppleScriptFile` +- Input: + - `path`: string (path to the AppleScript file) +- Executes an AppleScript from a file and returns the result +- Returns "AppleScript Result: " if the script returns a value +- Returns "AppleScript file executed successfully (no result returned): " if the script completes without returning a value + ## Security Considerations This server requires full accessibility permissions in System Preferences to control your mouse and keyboard. Be careful when running it and only connect trusted MCP clients. diff --git a/Sources/swift-mcp-gui/Tools/AppleScript/ExecuteAppleScriptFileTool.swift b/Sources/swift-mcp-gui/Tools/AppleScript/ExecuteAppleScriptFileTool.swift new file mode 100644 index 0000000..c869530 --- /dev/null +++ b/Sources/swift-mcp-gui/Tools/AppleScript/ExecuteAppleScriptFileTool.swift @@ -0,0 +1,37 @@ +import Foundation +import MCP +import SwiftAutoGUI + +struct ExecuteAppleScriptFileTool { + static func register(in registry: ToolRegistry) { + let tool = Tool( + name: "executeAppleScriptFile", + description: "Execute AppleScript from a file", + inputSchema: .object([ + "type": .string("object"), + "properties": .object([ + "path": .object(["type": .string("string"), "description": .string("Path to the AppleScript file")]) + ]), + "required": .array([.string("path")]) + ]) + ) + + registry.registerTool(definition: tool) { arguments in + let parser = ParameterParser(arguments: arguments) + + do { + let path = try parser.parseString("path") + + let result = try SwiftAutoGUI.executeAppleScriptFile(path) + + if let resultString = result { + return .init(content: [.text("AppleScript Result: \(resultString)")], isError: false) + } else { + return .init(content: [.text("AppleScript file executed successfully (no result returned): \(path)")], isError: false) + } + } catch { + return .init(content: [.text("Failed to execute AppleScript file: \(error.localizedDescription)")], isError: true) + } + } + } +} \ No newline at end of file diff --git a/Sources/swift-mcp-gui/Tools/AppleScript/ExecuteAppleScriptTool.swift b/Sources/swift-mcp-gui/Tools/AppleScript/ExecuteAppleScriptTool.swift new file mode 100644 index 0000000..66218c7 --- /dev/null +++ b/Sources/swift-mcp-gui/Tools/AppleScript/ExecuteAppleScriptTool.swift @@ -0,0 +1,37 @@ +import Foundation +import MCP +import SwiftAutoGUI + +struct ExecuteAppleScriptTool { + static func register(in registry: ToolRegistry) { + let tool = Tool( + name: "executeAppleScript", + description: "Execute AppleScript code directly", + inputSchema: .object([ + "type": .string("object"), + "properties": .object([ + "script": .object(["type": .string("string"), "description": .string("AppleScript code to execute")]) + ]), + "required": .array([.string("script")]) + ]) + ) + + registry.registerTool(definition: tool) { arguments in + let parser = ParameterParser(arguments: arguments) + + do { + let script = try parser.parseString("script") + + let result = try SwiftAutoGUI.executeAppleScript(script) + + if let resultString = result { + return .init(content: [.text("AppleScript Result: \(resultString)")], isError: false) + } else { + return .init(content: [.text("AppleScript executed successfully (no result returned)")], isError: false) + } + } catch { + return .init(content: [.text("Failed to execute AppleScript: \(error.localizedDescription)")], isError: true) + } + } + } +} \ No newline at end of file diff --git a/Sources/swift-mcp-gui/Tools/ToolRegistry.swift b/Sources/swift-mcp-gui/Tools/ToolRegistry.swift index 0bdbb1f..e9e91ca 100644 --- a/Sources/swift-mcp-gui/Tools/ToolRegistry.swift +++ b/Sources/swift-mcp-gui/Tools/ToolRegistry.swift @@ -33,5 +33,7 @@ class ToolRegistry { CaptureScreenTool.register(in: self) CaptureRegionTool.register(in: self) SaveScreenshotTool.register(in: self) + ExecuteAppleScriptTool.register(in: self) + ExecuteAppleScriptFileTool.register(in: self) } } \ No newline at end of file diff --git a/Tests/swift-mcp-guiTests/Tools/AppleScriptToolsTests.swift b/Tests/swift-mcp-guiTests/Tools/AppleScriptToolsTests.swift new file mode 100644 index 0000000..56981e6 --- /dev/null +++ b/Tests/swift-mcp-guiTests/Tools/AppleScriptToolsTests.swift @@ -0,0 +1,248 @@ +import Testing +import MCP +@testable import swift_mcp_gui + +@Suite("AppleScript Tools Tests") +struct AppleScriptToolsTests { + let toolRegistry: ToolRegistry + + init() { + self.toolRegistry = ToolRegistry() + ExecuteAppleScriptTool.register(in: toolRegistry) + ExecuteAppleScriptFileTool.register(in: toolRegistry) + } + + @Test("Execute AppleScript tool registration") + func executeAppleScriptToolRegistration() { + let tools = toolRegistry.listTools() + #expect(tools.contains { $0.name == "executeAppleScript" }) + + let tool = tools.first { $0.name == "executeAppleScript" } + #expect(tool != nil) + #expect(tool?.description == "Execute AppleScript code directly") + } + + @Test("Execute AppleScript file tool registration") + func executeAppleScriptFileToolRegistration() { + let tools = toolRegistry.listTools() + #expect(tools.contains { $0.name == "executeAppleScriptFile" }) + + let tool = tools.first { $0.name == "executeAppleScriptFile" } + #expect(tool != nil) + #expect(tool?.description == "Execute AppleScript from a file") + } + + @Test("Execute AppleScript tool with valid script") + func executeAppleScriptToolWithValidScript() async throws { + // Simple AppleScript that returns a value + let arguments: Value = .object([ + "script": .string("return \"Hello from AppleScript\"") + ]) + + let result = try await toolRegistry.execute(name: "executeAppleScript", arguments: arguments) + + // Note: This test might succeed or fail depending on AppleScript permissions + // In sandboxed environments or CI/CD, it might fail due to lack of automation permissions + #expect(result.content.count > 0) + + if result.isError != true { + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("AppleScript Result:") || text.contains("AppleScript executed successfully") + } + return false + } != nil) + } + } + + @Test("Execute AppleScript tool with result") + func executeAppleScriptToolWithResult() async throws { + // AppleScript that returns a calculated value + let arguments: Value = .object([ + "script": .string("return 2 + 2") + ]) + + let result = try await toolRegistry.execute(name: "executeAppleScript", arguments: arguments) + + // Note: This test might succeed or fail depending on AppleScript permissions + #expect(result.content.count > 0) + + if result.isError != true { + #expect(result.content.first { + if case .text(let text) = $0 { + // The result should contain "4" or similar + return text.contains("AppleScript Result:") || text.contains("no result returned") + } + return false + } != nil) + } + } + + @Test("Execute AppleScript tool without result") + func executeAppleScriptToolWithoutResult() async throws { + // AppleScript that doesn't return a value + let arguments: Value = .object([ + "script": .string("beep") + ]) + + let result = try await toolRegistry.execute(name: "executeAppleScript", arguments: arguments) + + // Note: This test might succeed or fail depending on AppleScript permissions + if result.isError != true { + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("no result returned") || text.contains("AppleScript Result:") + } + return false + } != nil) + } + } + + @Test("Execute AppleScript tool with missing script parameter") + func executeAppleScriptToolMissingScript() async throws { + let arguments: Value = .object([:]) + + let result = try await toolRegistry.execute(name: "executeAppleScript", arguments: arguments) + #expect(result.isError == true) + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("Missing parameter: script") + } + return false + } != nil) + } + + @Test("Execute AppleScript tool with invalid parameter type") + func executeAppleScriptToolInvalidParameterType() async throws { + let arguments: Value = .object([ + "script": .int(123) // Should be string + ]) + + let result = try await toolRegistry.execute(name: "executeAppleScript", arguments: arguments) + #expect(result.isError == true) + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("Invalid parameter script: expected string") + } + return false + } != nil) + } + + @Test("Execute AppleScript file tool with missing path parameter") + func executeAppleScriptFileToolMissingPath() async throws { + let arguments: Value = .object([:]) + + let result = try await toolRegistry.execute(name: "executeAppleScriptFile", arguments: arguments) + #expect(result.isError == true) + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("Missing parameter: path") + } + return false + } != nil) + } + + @Test("Execute AppleScript file tool with invalid parameter type") + func executeAppleScriptFileToolInvalidParameterType() async throws { + let arguments: Value = .object([ + "path": .bool(true) // Should be string + ]) + + let result = try await toolRegistry.execute(name: "executeAppleScriptFile", arguments: arguments) + #expect(result.isError == true) + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("Invalid parameter path: expected string") + } + return false + } != nil) + } + + @Test("Execute AppleScript file tool with non-existent file") + func executeAppleScriptFileToolNonExistentFile() async throws { + let arguments: Value = .object([ + "path": .string("/path/to/non/existent/script.applescript") + ]) + + let result = try await toolRegistry.execute(name: "executeAppleScriptFile", arguments: arguments) + #expect(result.isError == true) + #expect(result.content.first { + if case .text(let text) = $0 { + return text.contains("Failed to execute AppleScript file") + } + return false + } != nil) + } + + @Test("Execute AppleScript tool schema validation") + func executeAppleScriptToolSchema() { + let tools = toolRegistry.listTools() + let tool = tools.first { $0.name == "executeAppleScript" }! + + // Verify the input schema structure + if case .object(let schema) = tool.inputSchema { + if case .string(let typeValue) = schema["type"] { + #expect(typeValue == "object") + } + + if case .object(let properties) = schema["properties"] { + // Check script parameter + if case .object(let scriptProp) = properties["script"] { + if case .string(let typeValue) = scriptProp["type"] { + #expect(typeValue == "string") + } + if case .string(let descValue) = scriptProp["description"] { + #expect(descValue == "AppleScript code to execute") + } + } + + // Check required parameters + if case .array(let required) = schema["required"] { + #expect(required.count == 1) + #expect(required.contains { + if case .string(let value) = $0 { + return value == "script" + } + return false + }) + } + } + } + } + + @Test("Execute AppleScript file tool schema validation") + func executeAppleScriptFileToolSchema() { + let tools = toolRegistry.listTools() + let tool = tools.first { $0.name == "executeAppleScriptFile" }! + + // Verify the input schema structure + if case .object(let schema) = tool.inputSchema { + if case .string(let typeValue) = schema["type"] { + #expect(typeValue == "object") + } + + if case .object(let properties) = schema["properties"] { + // Check path parameter + if case .object(let pathProp) = properties["path"] { + if case .string(let typeValue) = pathProp["type"] { + #expect(typeValue == "string") + } + if case .string(let descValue) = pathProp["description"] { + #expect(descValue == "Path to the AppleScript file") + } + } + + // Check required parameters + if case .array(let required) = schema["required"] { + #expect(required.count == 1) + #expect(required.contains { + if case .string(let value) = $0 { + return value == "path" + } + return false + }) + } + } + } + } +} \ No newline at end of file