Merge pull request #42 from openai/contrib-refactor

Add support and guidelines for contributing computers

Author: ibigio

.gitignore

@@ -1,3 +1,4 @@
 __pycache__/
 .env
-.venv/
+.venv/
+env/

README.md

@@ -74,7 +74,7 @@ python simple_cua_loop.py
 
 ## Computer Environments
 
-CUA can work with any `Computer` environment that can handle the [CUA actions](https://platform.openai.com/docs/api-reference/responses/object#responses/object-output):
+CUA can work with any `Computer` environment that can handle the [CUA actions](https://platform.openai.com/docs/api-reference/responses/object#responses/object-output) (plus a few extra):
 
 | Action                             | Example                         |
 | ---------------------------------- | ------------------------------- |
@@ -109,6 +109,15 @@ For example, to run the sample app with the `Docker` computer environment, you c
 python cli.py --show --computer docker
 ```
 
+#### Contributed Computers
+
+| Computer | Option | Type | Description | Requirements |
+| -------- | ------ | ---- | ----------- | ------------ |
+| `tbd`    | tbd    | tbd  | tbd         | tbd          |
+
+> [!NOTE]  
+> If you've implemented a new computer, please add it to the "Contributed Computers" section of the README.md file. Clearly indicate any auth / signup requirements. See the [Contributing](#contributing) section for more details.
+
 ### Docker Setup
 
 If you want to run the sample app with the `Docker` computer environment, you need to build and run a local Docker container.
@@ -152,3 +161,98 @@ However, if you pass in any `tools` that are also defined in your `Computer` met
 This repository provides example implementations with basic safety measures in place.
 
 We recommend reviewing the best practices outlined in our [guide](https://platform.openai.com/docs/guides/tools-computer-use#risks-and-safety), and making sure you understand the risks involved with using this tool.
+
+# Contributing
+
+## Computers
+
+To contribute a new computer, you'll need to implement it, test it, and submit a PR. Please follow the steps below:
+
+### 1. Implement your computer
+
+You will create or modify the following files (and only these files):
+
+| File                                        | Updates            |
+| ------------------------------------------- | ------------------ |
+| `computers/contrib/[your_computer_name].py` | Add computer file. |
+| `computers/contrib/__init__.py`             | Add to imports.    |
+| `computers/config.py`                       | Add to config.     |
+| `README.md`                                 | Add to README.     |
+
+Create a new file in `computers/contrib/[your_computer_name].py` and define your computer class. Make sure to implement the methods defined in the `Computer` class – use the existing implementations as a reference.
+
+```python
+class YourComputerName:
+    def __init__(self):
+        pass
+
+    def screenshot(self):
+        # TODO: implement
+        pass
+
+    def click(self, x, y):
+        # TODO: implement
+        pass
+
+    # ... add other methods as needed
+```
+
+> [!NOTE]  
+> For playwright-based computers, make sure to subclass `BasePlaywrightComputer` in `computers/shared/base_playwright.py` – see `computers/default/browserbase.py` for an example.
+
+Import your new computer in the `computers/contrib/__init__.py`:
+
+```python
+# ... existing computer imports
+from .your_computer_name import YourComputerName
+```
+
+And add your new computer to the `computers_config` dictionary in `computers/config.py`:
+
+```python
+# ... existing computers_config
+"your_computer_name": YourComputerName,
+```
+
+Feel free to add your new computer to the "Contributed Computers" section of the README.md file. Clearly indicate any auth / signup requirements.
+
+### 2. Test your computer
+
+Test your new computer (with the CLI). Make sure:
+
+- Basic search / navigation works.
+- Any setup / teardown is handled correctly.
+- Test e2e with a few different tasks.
+
+Potential gotchas (See `default` computers for reference):
+
+- Scrolling, dragging, and control/command keys.
+- Resource allocation and teardown.
+- Auth / signup requirements.
+
+### 3. Submit a PR
+
+Your PR should clearly define the following:
+
+- Title: `[contrib] Add computer: <your_computer_name>`
+- Description:
+
+```
+# Add computer: <your_computer_name>
+
+#### Affiliations
+
+What organization / company / institution are you affiliated with?
+
+#### Computer Description
+
+- Computer type (e.g. browser, linux)
+
+#### Testing Plan
+
+- Signup steps.
+- Auth steps.
+- Sample queries.
+```
+
+Thank you for your contribution! Please follow all of the above guidelines. Failure to do so may result in your PR being rejected.

agent/agent.py

@@ -33,12 +33,13 @@ def __init__(
         self.acknowledge_safety_check_callback = acknowledge_safety_check_callback
 
         if computer:
+            dimensions = computer.get_dimensions()
             self.tools += [
                 {
                     "type": "computer-preview",
-                    "display_width": computer.dimensions[0],
-                    "display_height": computer.dimensions[1],
-                    "environment": computer.environment,
+                    "display_width": dimensions[0],
+                    "display_height": dimensions[1],
+                    "environment": computer.get_environment(),
                 },
             ]
 
@@ -102,7 +103,7 @@ def handle_item(self, item):
             }
 
             # additional URL safety checks for browser environments
-            if self.computer.environment == "browser":
+            if self.computer.get_environment() == "browser":
                 current_url = self.computer.get_current_url()
                 check_blocklisted_url(current_url)
                 call_output["output"]["current_url"] = current_url

cli.py

@@ -1,12 +1,9 @@
 import argparse
 from agent.agent import Agent
-from computers import (
-    BrowserbaseBrowser,
-    ScrapybaraBrowser,
-    ScrapybaraUbuntu,
-    LocalPlaywrightComputer,
-    DockerComputer,
-)
+from computers.config import *
+from computers.default import *
+from computers import computers_config
+
 
 def acknowledge_safety_check_callback(message: str) -> bool:
     response = input(
@@ -21,13 +18,7 @@ def main():
     )
     parser.add_argument(
         "--computer",
-        choices=[
-            "local-playwright",
-            "docker",
-            "browserbase",
-            "scrapybara-browser",
-            "scrapybara-ubuntu",
-        ],
+        choices=computers_config.keys(),
         help="Choose the computer environment to use.",
         default="local-playwright",
     )
@@ -54,16 +45,7 @@ def main():
         default="https://bing.com",
     )
     args = parser.parse_args()
-
-    computer_mapping = {
-        "local-playwright": LocalPlaywrightComputer,
-        "docker": DockerComputer,
-        "browserbase": BrowserbaseBrowser,
-        "scrapybara-browser": ScrapybaraBrowser,
-        "scrapybara-ubuntu": ScrapybaraUbuntu,
-    }
-
-    ComputerClass = computer_mapping[args.computer]
+    ComputerClass = computers_config[args.computer]
 
     with ComputerClass() as computer:
         agent = Agent(
@@ -72,7 +54,6 @@ def main():
         )
         items = []
 
-
         if args.computer in ["browserbase", "local-playwright"]:
             if not args.start_url.startswith("http"):
                 args.start_url = "https://" + args.start_url
@@ -81,7 +62,7 @@ def main():
         while True:
             try:
                 user_input = args.input or input("> ")
-                if user_input == 'exit':
+                if user_input == "exit":
                     break
             except EOFError as e:
                 print(f"An error occurred: {e}")

computers/__init__.py

@@ -1,5 +1,11 @@
+from . import default
+from . import contrib
 from .computer import Computer
-from .browserbase import BrowserbaseBrowser
-from .local_playwright import LocalPlaywrightComputer
-from .docker import DockerComputer
-from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu
+from .config import computers_config
+
+__all__ = [
+    "default",
+    "contrib",
+    "Computer",
+    "computers_config",
+]

computers/computer.py

@@ -4,10 +4,9 @@
 class Computer(Protocol):
     """Defines the 'shape' (methods/properties) our loop expects."""
 
-    @property
-    def environment(self) -> Literal["windows", "mac", "linux", "browser"]: ...
-    @property
-    def dimensions(self) -> tuple[int, int]: ...
+    def get_environment(self) -> Literal["windows", "mac", "linux", "browser"]: ...
+
+    def get_dimensions(self) -> tuple[int, int]: ...
 
     def screenshot(self) -> str: ...
 

computers/contrib/__init__.py

@@ -0,0 +1,4 @@
+from .browserbase import BrowserbaseBrowser
+from .local_playwright import LocalPlaywrightBrowser
+from .docker import DockerComputer
+from .scrapybara import ScrapybaraBrowser, ScrapybaraUbuntu

computers/default/__init__.py

@@ -1,7 +1,7 @@
 import os
 from typing import Tuple, Dict, List, Union, Optional
 from playwright.sync_api import Browser, Page, BrowserContext, Error as PlaywrightError
-from .base_playwright import BasePlaywrightComputer
+from ..shared.base_playwright import BasePlaywrightComputer
 from browserbase import Browserbase
 from dotenv import load_dotenv
 import base64
@@ -18,6 +18,9 @@ class BrowserbaseBrowser(BasePlaywrightComputer):
     Make sure to include this tool in your configuration when using the Browserbase computer.
     """
 
+    def get_dimensions(self):
+        return self.dimensions
+
     def __init__(
         self,
         width: int = 1024,
@@ -75,8 +78,7 @@ def _get_browser_and_page(self) -> Tuple[Browser, Page]:
 
         # Connect to the remote session
         browser = self._playwright.chromium.connect_over_cdp(
-            self.session.connect_url,
-            timeout=60000
+            self.session.connect_url, timeout=60000
         )
         context = browser.contexts[0]
 
@@ -85,7 +87,8 @@ def _get_browser_and_page(self) -> Tuple[Browser, Page]:
 
         # Only add the init script if virtual_mouse is True
         if self.virtual_mouse:
-            context.add_init_script("""
+            context.add_init_script(
+                """
             // Only run in the top frame
             if (window.self === window.top) {
                 function initCursor() {
@@ -126,7 +129,8 @@ def _get_browser_and_page(self) -> Tuple[Browser, Page]:
                     }
                 });
             }
-            """)
+            """
+            )
 
         page = context.pages[0]
         page.on("close", self._handle_page_close)
@@ -184,12 +188,13 @@ def screenshot(self) -> str:
             cdp_session = self._page.context.new_cdp_session(self._page)
 
             # Capture screenshot using CDP
-            result = cdp_session.send("Page.captureScreenshot", {
-                "format": "png",
-                "fromSurface": True
-            })
+            result = cdp_session.send(
+                "Page.captureScreenshot", {"format": "png", "fromSurface": True}
+            )
 
-            return result['data']
+            return result["data"]
         except PlaywrightError as error:
-            print(f"CDP screenshot failed, falling back to standard screenshot: {error}")
+            print(
+                f"CDP screenshot failed, falling back to standard screenshot: {error}"
+            )
             return super().screenshot()

computers/browserbase.py renamed to computers/default/browserbase.py

@@ -4,8 +4,11 @@
 
 
 class DockerComputer:
-    environment = "linux"
-    dimensions = (1280, 720)  # Default fallback; will be updated in __enter__.
+    def get_environment(self):
+        return "linux"
+
+    def get_dimensions(self):
+        return (1280, 720)  # Default fallback; will be updated in __enter__.
 
     def __init__(
         self,
@@ -162,5 +165,10 @@ def drag(self, path: list[dict[str, int]]) -> None:
             f"DISPLAY={self.display} xdotool mousemove {start_x} {start_y} mousedown 1"
         )
         for point in path[1:]:
-            self._exec(f"DISPLAY={self.display} xdotool mousemove {point['x']} {point['y']}")
+            self._exec(
+                f"DISPLAY={self.display} xdotool mousemove {point['x']} {point['y']}"
+            )
         self._exec(f"DISPLAY={self.display} xdotool mouseup 1")
+
+    def get_current_url(self):
+        return None