From 3328acc8fec4616896d3f85fe6809853625ca23d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20Hvilsh=C3=B8j?= Date: Tue, 14 Jan 2025 10:37:40 +0100 Subject: [PATCH 1/4] feat: add cli to test other (custom) endpoints Should make it easier to develop and verify, e.g., public endpoints --- encord_agents/cli/test.py | 120 +++++++++++++++++++------------ encord_agents/core/data_model.py | 4 +- 2 files changed, 77 insertions(+), 47 deletions(-) diff --git a/encord_agents/cli/test.py b/encord_agents/cli/test.py index 1d0d576..1cd355c 100644 --- a/encord_agents/cli/test.py +++ b/encord_agents/cli/test.py @@ -3,12 +3,19 @@ """ import os +import re +import sys +import requests +import rich +import typer from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from rich.table import Table from typer import Argument, Option, Typer from typing_extensions import Annotated +from encord_agents import FrameData + app = Typer( name="test", help="Utility for testing agents", @@ -16,51 +23,17 @@ no_args_is_help=True, ) +EDITOR_URL_PARTS_REGEX = r"https:\/\/app.encord.com\/label_editor\/(?P.*?)\/(?P[\w\d]{8}-[\w\d]{4}-[\w\d]{4}-[\w\d]{4}-[\w\d]{12})(/(?P\d+))?\??" -@app.command( - "local", - short_help="Hit a localhost agents endpoint for testing", -) -def local( - target: Annotated[ - str, - Argument(help="Name of the localhost endpoint to hit ('http://localhost/{target}')"), - ], - url: Annotated[str, Argument(help="Url copy/pasted from label editor")], - port: Annotated[int, Option(help="Local host port to hit")] = 8080, -) -> None: - """Hit a localhost agents endpoint for testing an agent by copying the url from the Encord Label Editor over. - - Given - - - A url of the form [blue]`https://app.encord.com/label_editor/[green]{project_hash}[/green]/[green]{data_hash}[/green]/[green]{frame}[/green]`[/blue] - - A [green]target[/green] endpoint - - A [green]port[/green] (optional) - - The url [blue]http://localhost:[green]{port}[/green]/[green]{target}[/green][/blue] will be hit with a post request containing: - { - "projectHash": "[green]{project_hash}[/green]", - "dataHash": "[green]{data_hash}[/green]", - "frame": [green]frame[/green] or 0 - } - """ - import re - import sys - from pprint import pprint - - import requests - import rich - import typer - - parts_regex = r"https:\/\/app.encord.com\/label_editor\/(?P.*?)\/(?P[\w\d]{8}-[\w\d]{4}-[\w\d]{4}-[\w\d]{4}-[\w\d]{12})(/(?P\d+))?\??" +def parse_editor_url(editor_url: str) -> FrameData: try: - match = re.match(parts_regex, url) + match = re.match(EDITOR_URL_PARTS_REGEX, editor_url) if match is None: raise typer.Abort() - payload = match.groupdict() payload["frame"] = payload["frame"] or 0 + return FrameData.model_validate(payload) except Exception: rich.print( """Could not match url to the expected format. @@ -70,14 +43,13 @@ def local( ) raise typer.Abort() - if target and not target[0] == "/": - target = f"/{target}" +def hit_endpoint(endpoint: str, payload: FrameData) -> None: with requests.Session() as sess: request = requests.Request( "POST", - f"http://localhost:{port}{target}", - json=payload, + endpoint, + json=payload.model_dump(mode="json", by_alias=True), headers={"Content-type": "application/json"}, ) prepped = request.prepare() @@ -109,9 +81,7 @@ def local( table.add_section() table.add_row("[green]Utilities[/green]") - editor_url = ( - f"https://app.encord.com/label_editor/{payload['projectHash']}/{payload['dataHash']}/{payload['frame']}" - ) + editor_url = f"https://app.encord.com/label_editor/{payload.project_hash}/{payload.data_hash}/{payload.frame}" table.add_row("label editor", editor_url) headers = ["'{0}: {1}'".format(k, v) for k, v in prepped.headers.items()] @@ -120,3 +90,63 @@ def local( table.add_row("curl", curl_command) rich.print(table) + + +@app.command("custom", short_help="Hit a custom endpoint for testing purposes") +def custom( + endpoint: Annotated[str, Argument(help="Endpoint to hit with json payload")], + editor_url: Annotated[str, Argument(help="Url copy/pasted from label editor")], +) -> None: + """ + Hit a custom agents endpoint for testing an editor agent by copying the url from the Encord Label Editor. + + Given + + - The endpoint you wish to test + - An editor url of the form [blue]`https://app.encord.com/label_editor/[green]{project_hash}[/green]/[green]{data_hash}[/green]/[green]{frame}[/green]`[/blue] + - A [green]port[/green] (optional) + + The url [blue]http://localhost:[green]{port}[/green]/[green]{target}[/green][/blue] will be hit with a post request containing: + { + "projectHash": "[green]{project_hash}[/green]", + "dataHash": "[green]{data_hash}[/green]", + "frame": [green]frame[/green] or 0 + } + """ + payload = parse_editor_url(editor_url) + hit_endpoint(endpoint, payload) + + +@app.command( + "local", + short_help="Hit a localhost agents endpoint for testing", +) +def local( + target: Annotated[ + str, + Argument(help="Name of the localhost endpoint to hit ('http://localhost/{target}')"), + ], + editor_url: Annotated[str, Argument(help="Url copy/pasted from label editor")], + port: Annotated[int, Option(help="Local host port to hit")] = 8080, +) -> None: + """Hit a localhost agents endpoint for testing an agent by copying the url from the Encord Label Editor over. + + Given + + - An editor url of the form [blue]`https://app.encord.com/label_editor/[green]{project_hash}[/green]/[green]{data_hash}[/green]/[green]{frame}[/green]`[/blue] + - A [green]port[/green] (optional) + + The url [blue]http://localhost:[green]{port}[/green]/[green]{target}[/green][/blue] will be hit with a post request containing: + { + "projectHash": "[green]{project_hash}[/green]", + "dataHash": "[green]{data_hash}[/green]", + "frame": [green]frame[/green] or 0 + } + """ + payload = parse_editor_url(editor_url) + + if target and not target[0] == "/": + target = f"/{target}" + endpoint = f"http://localhost:{port}{target}" + + hit_endpoint(endpoint, payload) diff --git a/encord_agents/core/data_model.py b/encord_agents/core/data_model.py index 6701d84..f3037cc 100644 --- a/encord_agents/core/data_model.py +++ b/encord_agents/core/data_model.py @@ -45,11 +45,11 @@ class FrameData(BaseModel): Holds the data sent from the Encord Label Editor at the time of triggering the agent. """ - project_hash: UUID = Field(validation_alias="projectHash") + project_hash: UUID = Field(alias="projectHash") """ The identifier of the given project. """ - data_hash: UUID = Field(validation_alias="dataHash") + data_hash: UUID = Field(alias="dataHash") """ The identifier of the given data asset. """ From 3e0f2ae6d01305fa9f124588a58317ce62cc5e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20Hvilsh=C3=B8j?= Date: Wed, 15 Jan 2025 11:25:27 +0100 Subject: [PATCH 2/4] fix: adapt to the us vs rest domain in cli test --- encord_agents/cli/test.py | 29 +++++++++++++++++++---------- encord_agents/core/constants.py | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/encord_agents/cli/test.py b/encord_agents/cli/test.py index 1cd355c..0976549 100644 --- a/encord_agents/cli/test.py +++ b/encord_agents/cli/test.py @@ -15,6 +15,7 @@ from typing_extensions import Annotated from encord_agents import FrameData +from encord_agents.core.constants import EDITOR_URL_PARTS_REGEX app = Typer( name="test", @@ -23,28 +24,36 @@ no_args_is_help=True, ) -EDITOR_URL_PARTS_REGEX = r"https:\/\/app.encord.com\/label_editor\/(?P.*?)\/(?P[\w\d]{8}-[\w\d]{4}-[\w\d]{4}-[\w\d]{4}-[\w\d]{12})(/(?P\d+))?\??" +def parse_editor_url(editor_url: str) -> tuple[FrameData, str]: + """ + Reads project_hash, data_hash, frame and domain from the editor url. + + Args: + - editor_url: The url obtained from the Label Editor. -def parse_editor_url(editor_url: str) -> FrameData: + Returns: + The FrameData object and the domain of the url. + """ try: match = re.match(EDITOR_URL_PARTS_REGEX, editor_url) if match is None: raise typer.Abort() payload = match.groupdict() + domain = payload.pop("domain") payload["frame"] = payload["frame"] or 0 - return FrameData.model_validate(payload) + return FrameData.model_validate(payload), domain except Exception: rich.print( """Could not match url to the expected format. -Format is expected to be [blue]https://app.encord.com/label_editor/[magenta]{project_hash}[/magenta]/[magenta]{data_hash}[/magenta](/[magenta]{frame}[/magenta])[/blue] +Format is expected to be [blue]https://app.(us.)?encord.com/label_editor/[magenta]{project_hash}[/magenta]/[magenta]{data_hash}[/magenta](/[magenta]{frame}[/magenta])[/blue] """, file=sys.stderr, ) raise typer.Abort() -def hit_endpoint(endpoint: str, payload: FrameData) -> None: +def hit_endpoint(endpoint: str, payload: FrameData, domain: str) -> None: with requests.Session() as sess: request = requests.Request( "POST", @@ -81,7 +90,7 @@ def hit_endpoint(endpoint: str, payload: FrameData) -> None: table.add_section() table.add_row("[green]Utilities[/green]") - editor_url = f"https://app.encord.com/label_editor/{payload.project_hash}/{payload.data_hash}/{payload.frame}" + editor_url = f"{domain}/label_editor/{payload.project_hash}/{payload.data_hash}/{payload.frame}" table.add_row("label editor", editor_url) headers = ["'{0}: {1}'".format(k, v) for k, v in prepped.headers.items()] @@ -113,8 +122,8 @@ def custom( "frame": [green]frame[/green] or 0 } """ - payload = parse_editor_url(editor_url) - hit_endpoint(endpoint, payload) + payload, domain = parse_editor_url(editor_url) + hit_endpoint(endpoint, payload, domain) @app.command( @@ -143,10 +152,10 @@ def local( "frame": [green]frame[/green] or 0 } """ - payload = parse_editor_url(editor_url) + payload, domain = parse_editor_url(editor_url) if target and not target[0] == "/": target = f"/{target}" endpoint = f"http://localhost:{port}{target}" - hit_endpoint(endpoint, payload) + hit_endpoint(endpoint, payload, domain) diff --git a/encord_agents/core/constants.py b/encord_agents/core/constants.py index 53f5f89..25b9042 100644 --- a/encord_agents/core/constants.py +++ b/encord_agents/core/constants.py @@ -1,3 +1,4 @@ ENCORD_DOMAIN_REGEX = ( r"^https:\/\/(?:(?:cord-ai-development--[\w\d]+-[\w\d]+\.web.app)|(?:(?:dev|staging|app)\.(us\.)?encord\.com))$" ) +EDITOR_URL_PARTS_REGEX = r"(?https:\/\/app.(us\.)?encord.com)\/label_editor\/(?P.*?)\/(?P[\w\d]{8}-[\w\d]{4}-[\w\d]{4}-[\w\d]{4}-[\w\d]{12})(\/(?P\d+))?\??" From 0e1c0631e2ba84b79e35744e249d21c3f7066aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20Hvilsh=C3=B8j?= Date: Tue, 14 Jan 2025 15:17:37 +0100 Subject: [PATCH 3/4] fix(mypy): naming --- ...ion.py => fastapi_frame_classification.py} | 0 ...on.py => fastapi_object_classification.py} | 0 ...ounding_box.py => gcp_add_bounding_box.py} | 0 ...ication.py => gcp_frame_classification.py} | 0 ...cation.py => gcp_object_classification.py} | 0 docs/editor_agents/examples/index.md | 48 +++++++++---------- 6 files changed, 24 insertions(+), 24 deletions(-) rename docs/code_examples/fastapi/{frame_classification.py => fastapi_frame_classification.py} (100%) rename docs/code_examples/fastapi/{object_classification.py => fastapi_object_classification.py} (100%) rename docs/code_examples/gcp/{add_bounding_box.py => gcp_add_bounding_box.py} (100%) rename docs/code_examples/gcp/{frame_classification.py => gcp_frame_classification.py} (100%) rename docs/code_examples/gcp/{object_classification.py => gcp_object_classification.py} (100%) diff --git a/docs/code_examples/fastapi/frame_classification.py b/docs/code_examples/fastapi/fastapi_frame_classification.py similarity index 100% rename from docs/code_examples/fastapi/frame_classification.py rename to docs/code_examples/fastapi/fastapi_frame_classification.py diff --git a/docs/code_examples/fastapi/object_classification.py b/docs/code_examples/fastapi/fastapi_object_classification.py similarity index 100% rename from docs/code_examples/fastapi/object_classification.py rename to docs/code_examples/fastapi/fastapi_object_classification.py diff --git a/docs/code_examples/gcp/add_bounding_box.py b/docs/code_examples/gcp/gcp_add_bounding_box.py similarity index 100% rename from docs/code_examples/gcp/add_bounding_box.py rename to docs/code_examples/gcp/gcp_add_bounding_box.py diff --git a/docs/code_examples/gcp/frame_classification.py b/docs/code_examples/gcp/gcp_frame_classification.py similarity index 100% rename from docs/code_examples/gcp/frame_classification.py rename to docs/code_examples/gcp/gcp_frame_classification.py diff --git a/docs/code_examples/gcp/object_classification.py b/docs/code_examples/gcp/gcp_object_classification.py similarity index 100% rename from docs/code_examples/gcp/object_classification.py rename to docs/code_examples/gcp/gcp_object_classification.py diff --git a/docs/editor_agents/examples/index.md b/docs/editor_agents/examples/index.md index 7dead05..10957b1 100644 --- a/docs/editor_agents/examples/index.md +++ b/docs/editor_agents/examples/index.md @@ -128,7 +128,7 @@ An agent that transforms a labeling task from Figure A to Figure B, as shown bel ??? "The full code for `agent.py`" - [agent.py](../../code_examples/gcp/frame_classification.py) linenums:1 + [agent.py](../../code_examples/gcp/gcp_frame_classification.py) linenums:1 Let's go through the code section by section. @@ -139,13 +139,13 @@ First, we import dependencies and set up the Project: Make sure to insert your Project's hash here. -[agent.py](../../code_examples/gcp/frame_classification.py) lines:1-15 +[agent.py](../../code_examples/gcp/gcp_frame_classification.py) lines:1-15 Next, we create a data model and a system prompt based on the Project Ontology that will tell Claude how to structure its response: -[agent.py](../../code_examples/gcp/frame_classification.py) lines:18-29 +[agent.py](../../code_examples/gcp/gcp_frame_classification.py) lines:18-29 @@ -316,14 +316,14 @@ Next, we create a data model and a system prompt based on the Project Ontology t We also need an Anthropic API client to communicate with Claude: -[agent.py](../../code_examples/gcp/frame_classification.py) lines:32-33 +[agent.py](../../code_examples/gcp/gcp_frame_classification.py) lines:32-33 Finally, we define our editor agent: -[agent.py](../../code_examples/gcp/frame_classification.py) lines:36-65 +[agent.py](../../code_examples/gcp/gcp_frame_classification.py) lines:36-65 The agent: @@ -587,7 +587,7 @@ The goal is to be able to trigger an agent that takes a labeling task from Figur ??? "The full code for `agent.py`" - [agent.py](../../code_examples/gcp/object_classification.py) linenums:1 + [agent.py](../../code_examples/gcp/gcp_object_classification.py) linenums:1 @@ -597,7 +597,7 @@ For this, you will need to have your `` ready. -[agent.py](../../code_examples/gcp/object_classification.py) lines:1-14 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:1-14 @@ -605,7 +605,7 @@ Now that we have the project, we can extract the generic ontology object as well -[agent.py](../../code_examples/gcp/object_classification.py) lines:15-19 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:15-19 @@ -621,7 +621,7 @@ is only allowed to choose between the object types that are not of the generic o -[agent.py](../../code_examples/gcp/object_classification.py) lines:22-30 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:22-30 @@ -905,7 +905,7 @@ With the system prompt ready, we can instantiate an api client for Claude. -[agent.py](../../code_examples/gcp/object_classification.py) lines:33-34 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:33-34 @@ -913,7 +913,7 @@ Now, let's define the editor agent. -[agent.py](../../code_examples/gcp/object_classification.py) lines:38-46 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:38-46 @@ -927,7 +927,7 @@ Notice how the `crop` variable has a convenient `b64_encoding` method to produce -[agent.py](../../code_examples/gcp/object_classification.py) lines:47-60 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:47-60 @@ -938,7 +938,7 @@ If successful, the old generic object can be removed and the newly classified ob -[agent.py](../../code_examples/gcp/object_classification.py) lines:63-80 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:63-80 @@ -946,7 +946,7 @@ Finally, we'll save the labels with Encord. -[agent.py](../../code_examples/gcp/object_classification.py) lines:83-84 +[agent.py](../../code_examples/gcp/gcp_object_classification.py) lines:83-84 @@ -1039,7 +1039,7 @@ The goal is to trigger an agent that takes a labeling task from Figure A to Figu ??? "The full code for `main.py`" - [main.py](../../code_examples/fastapi/frame_classification.py) linenums:1 + [main.py](../../code_examples/fastapi/fastapi_frame_classification.py) linenums:1 Let us go through the code section by section. @@ -1047,7 +1047,7 @@ Let us go through the code section by section. First, we import dependencies and setup the FastAPI app with CORS middleware: -[main.py](../../code_examples/fastapi/frame_classification.py) lines:1-22 +[main.py](../../code_examples/fastapi/fastapi_frame_classification.py) lines:1-22 The CORS middleware is crucial as it allows the Encord platform to make requests to your API. @@ -1055,19 +1055,19 @@ The CORS middleware is crucial as it allows the Encord platform to make requests Next, we set up the Project and create a data model based on the Ontology: -[main.py](../../code_examples/fastapi/frame_classification.py) lines:25-27 +[main.py](../../code_examples/fastapi/fastapi_frame_classification.py) lines:25-27 We create the system prompt that tells Claude how to structure its response: -[main.py](../../code_examples/fastapi/frame_classification.py) lines:30-42 +[main.py](../../code_examples/fastapi/fastapi_frame_classification.py) lines:30-42 Finally, we define the endpoint to handle the classification: -[main.py](../../code_examples/fastapi/frame_classification.py) lines:45-75 +[main.py](../../code_examples/fastapi/fastapi_frame_classification.py) lines:45-75 The endpoint: @@ -1147,7 +1147,7 @@ The goal is to trigger an agent that takes a labeling task from Figure A to Figu ??? "The full code for `main.py`" - [main.py](../../code_examples/fastapi/object_classification.py) linenums:1 + [main.py](../../code_examples/fastapi/fastapi_object_classification.py) linenums:1 Let's walk through the key components. @@ -1155,25 +1155,25 @@ Let's walk through the key components. First, we setup the FastAPI app and CORS middleware: -[main.py](../../code_examples/fastapi/object_classification.py) lines:1-20 +[main.py](../../code_examples/fastapi/fastapi_object_classification.py) lines:1-20 Then we setup the client, Project, and extract the generic Ontology object: -[main.py](../../code_examples/fastapi/object_classification.py) lines:23-29 +[main.py](../../code_examples/fastapi/fastapi_object_classification.py) lines:23-29 We create the data model and system prompt for Claude: -[main.py](../../code_examples/fastapi/object_classification.py) lines:32-44 +[main.py](../../code_examples/fastapi/fastapi_object_classification.py) lines:32-44 Finally, we define our object classification endpoint: -[main.py](../../code_examples/fastapi/object_classification.py) lines:47-94 +[main.py](../../code_examples/fastapi/fastapi_object_classification.py) lines:47-94 The endpoint: From 5ccf384adb5804f7521500a13f84c93fe0cba646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20Hvilsh=C3=B8j?= Date: Tue, 14 Jan 2025 15:18:35 +0100 Subject: [PATCH 4/4] fix: exclude docs build for mypy --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 37f6ced..f050595 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ ignore = ["F401", "E402", "W291"] [tool.mypy] plugins = ['pydantic.mypy'] -exclude = ['docs/', 'encord_agents/core/ontology.py'] +exclude = ['docs/', 'encord_agents/core/ontology.py', 'site/'] python_version = "3.10" warn_unused_ignores = true disallow_untyped_calls = true