From 96a3dbfd1aa4e1e556fcf9f3deed2766f86b8409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20Hvilsh=C3=B8j?= <93145535+frederik-encord@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:49:53 +0100 Subject: [PATCH] chore: pre commit (#29) * fix: ruff formatting * fix: install pre-commit hook properly * chore: add gh action for linting / testing --- .github/workflows/pull-request.yml | 39 +++++++++++++++++++ .pre-commit-config.yaml | 2 + .../fastapi/frame_classification.py | 14 +++---- .../fastapi/object_classification.py | 12 +++--- docs/code_examples/gcp/add_bounding_box.py | 5 +-- .../code_examples/gcp/frame_classification.py | 9 ++--- .../gcp/object_classification.py | 7 ++-- docs/code_examples/tasks/prelabel_videos.py | 9 ++--- .../tasks/prioritize_by_data_title.py | 1 + .../prioritize_by_data_title_specific.py | 1 + docs/code_examples/tasks/twin_project.py | 7 ++-- .../tasks/wrong_stage_and_pathway_names.py | 1 + encord_agents/core/settings.py | 2 +- encord_agents/core/types.py | 2 - encord_agents/core/vision.py | 4 +- encord_agents/fastapi/dependencies.py | 8 +++- encord_agents/tasks/runner.py | 6 ++- 17 files changed, 84 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/pull-request.yml diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 0000000..7600e34 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,39 @@ +name: Pull Request workflow + +on: + pull_request: + workflow_dispatch: + +env: + PYTHON: 3.11.6 + +concurrency: + group: active-private-${{ github.ref }}-pr + cancel-in-progress: true + +jobs: + # This will cache the virtual env for the following jobs + setup-venv: + runs-on: ubuntu-latest + name: Set up a cached VENV + steps: + - uses: actions/checkout@v3 + + - name: Setup root poetry environment + uses: ./.github/actions/setup-root-poetry-environment + + pre-commit: + needs: [setup-venv] + name: Linting and type checking + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup root poetry environment + uses: ./.github/actions/setup-root-poetry-environment + + - name: Run linting, type checking + uses: pre-commit/action@v3.0.0 + with: + extra_args: "--all-files --hook-stage=push" + timeout-minutes: 60 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a916df0..9dc57dc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,4 +28,6 @@ repos: types_or: [python, pyi] language: system pass_filenames: false + default_stages: [pre-push] +default_install_hook_types: [pre-push] diff --git a/docs/code_examples/fastapi/frame_classification.py b/docs/code_examples/fastapi/frame_classification.py index 325fb4d..25d43f9 100644 --- a/docs/code_examples/fastapi/frame_classification.py +++ b/docs/code_examples/fastapi/frame_classification.py @@ -3,6 +3,11 @@ import numpy as np from anthropic import Anthropic from encord.objects.ontology_labels_impl import LabelRowV2 +from fastapi import Depends, FastAPI, Form +from fastapi.middleware.cors import CORSMiddleware +from numpy.typing import NDArray +from typing_extensions import Annotated + from encord_agents.core.data_model import Frame from encord_agents.core.ontology import OntologyDataModel from encord_agents.core.utils import get_user_client @@ -11,11 +16,6 @@ dep_label_row, dep_single_frame, ) -from numpy.typing import NDArray -from typing_extensions import Annotated - -from fastapi import Depends, FastAPI, Form -from fastapi.middleware.cors import CORSMiddleware # Initialize FastAPI app app = FastAPI() @@ -67,9 +67,7 @@ async def classify_frame( try: classifications = data_model(message.content[0].text) for clf in classifications: - clf.set_for_frames( - frame_data.frame, confidence=0.5, manual_annotation=False - ) + clf.set_for_frames(frame_data.frame, confidence=0.5, manual_annotation=False) lr.add_classification_instance(clf) except Exception: import traceback diff --git a/docs/code_examples/fastapi/object_classification.py b/docs/code_examples/fastapi/object_classification.py index 5169b42..ebf30a4 100644 --- a/docs/code_examples/fastapi/object_classification.py +++ b/docs/code_examples/fastapi/object_classification.py @@ -2,6 +2,10 @@ from anthropic import Anthropic from encord.objects.ontology_labels_impl import LabelRowV2 +from fastapi import Depends, FastAPI, Form +from fastapi.middleware.cors import CORSMiddleware +from typing_extensions import Annotated + from encord_agents.core.data_model import InstanceCrop from encord_agents.core.ontology import OntologyDataModel from encord_agents.core.utils import get_user_client @@ -10,10 +14,6 @@ dep_label_row, dep_object_crops, ) -from typing_extensions import Annotated - -from fastapi import Depends, FastAPI, Form -from fastapi.middleware.cors import CORSMiddleware # Initialize FastAPI app app = FastAPI() @@ -76,9 +76,7 @@ async def classify_objects( try: instance = data_model(message.content[0].text) - coordinates = crop.instance.get_annotation( - frame=frame_data.frame - ).coordinates + coordinates = crop.instance.get_annotation(frame=frame_data.frame).coordinates instance.set_for_frames( coordinates=coordinates, frames=frame_data.frame, diff --git a/docs/code_examples/gcp/add_bounding_box.py b/docs/code_examples/gcp/add_bounding_box.py index 9a97432..22f2125 100644 --- a/docs/code_examples/gcp/add_bounding_box.py +++ b/docs/code_examples/gcp/add_bounding_box.py @@ -17,6 +17,7 @@ from encord.objects.coordinates import BoundingBoxCoordinates from encord.objects.ontology_labels_impl import LabelRowV2 + from encord_agents.core.data_model import FrameData from encord_agents.gcp import editor_agent @@ -26,9 +27,7 @@ def add_bounding_box(frame_data: FrameData, label_row: LabelRowV2) -> None: ins = label_row.ontology_structure.objects[0].create_instance() ins.set_for_frames( frames=frame_data.frame, - coordinates=BoundingBoxCoordinates( - top_left_x=0.2, top_left_y=0.2, width=0.6, height=0.6 - ), + coordinates=BoundingBoxCoordinates(top_left_x=0.2, top_left_y=0.2, width=0.6, height=0.6), ) label_row.add_object_instance(ins) label_row.save() diff --git a/docs/code_examples/gcp/frame_classification.py b/docs/code_examples/gcp/frame_classification.py index 60197f1..5a37bb9 100644 --- a/docs/code_examples/gcp/frame_classification.py +++ b/docs/code_examples/gcp/frame_classification.py @@ -2,13 +2,14 @@ from anthropic import Anthropic from encord.objects.ontology_labels_impl import LabelRowV2 +from numpy.typing import NDArray +from typing_extensions import Annotated + from encord_agents.core.ontology import OntologyDataModel from encord_agents.core.utils import get_user_client from encord_agents.core.video import Frame from encord_agents.gcp import Depends, editor_agent from encord_agents.gcp.dependencies import FrameData, dep_single_frame -from numpy.typing import NDArray -from typing_extensions import Annotated client = get_user_client() project = client.get_project("") @@ -53,9 +54,7 @@ def agent( try: classifications = data_model(message.content[0].text) for clf in classifications: - clf.set_for_frames( - frame_data.frame, confidence=0.5, manual_annotation=False - ) + clf.set_for_frames(frame_data.frame, confidence=0.5, manual_annotation=False) lr.add_classification_instance(clf) except Exception: import traceback diff --git a/docs/code_examples/gcp/object_classification.py b/docs/code_examples/gcp/object_classification.py index ce5d3fb..4d074c8 100644 --- a/docs/code_examples/gcp/object_classification.py +++ b/docs/code_examples/gcp/object_classification.py @@ -2,11 +2,12 @@ from anthropic import Anthropic from encord.objects.ontology_labels_impl import LabelRowV2 +from typing_extensions import Annotated + from encord_agents.core.ontology import OntologyDataModel from encord_agents.core.utils import get_user_client from encord_agents.gcp import Depends, editor_agent from encord_agents.gcp.dependencies import FrameData, InstanceCrop, dep_object_crops -from typing_extensions import Annotated # User client client = get_user_client() @@ -62,9 +63,7 @@ def agent( try: instance = data_model(message.content[0].text) - coordinates = crop.instance.get_annotation( - frame=frame_data.frame - ).coordinates + coordinates = crop.instance.get_annotation(frame=frame_data.frame).coordinates instance.set_for_frames( coordinates=coordinates, frames=frame_data.frame, diff --git a/docs/code_examples/tasks/prelabel_videos.py b/docs/code_examples/tasks/prelabel_videos.py index a229288..a21916d 100644 --- a/docs/code_examples/tasks/prelabel_videos.py +++ b/docs/code_examples/tasks/prelabel_videos.py @@ -15,11 +15,12 @@ from encord.objects.coordinates import BoundingBoxCoordinates from encord.objects.ontology_labels_impl import LabelRowV2 from encord.project import Project +from numpy.typing import NDArray +from typing_extensions import Annotated + from encord_agents.core.data_model import Frame from encord_agents.tasks import Depends, Runner from encord_agents.tasks.dependencies import dep_video_iterator -from numpy.typing import NDArray -from typing_extensions import Annotated runner = Runner(project_hash="") @@ -64,9 +65,7 @@ def run_something( outputs = model(frame.content) for output in outputs: ins = ontology.objects[output.label].create_instance() - ins.set_for_frames( - frames=frame.frame, coordinates=output.coords, confidence=output.conf - ) + ins.set_for_frames(frames=frame.frame, coordinates=output.coords, confidence=output.conf) lr.add_object_instance(ins) diff --git a/docs/code_examples/tasks/prioritize_by_data_title.py b/docs/code_examples/tasks/prioritize_by_data_title.py index a20d7da..4fd0795 100644 --- a/docs/code_examples/tasks/prioritize_by_data_title.py +++ b/docs/code_examples/tasks/prioritize_by_data_title.py @@ -1,4 +1,5 @@ from encord.objects.ontology_labels_impl import LabelRowV2 + from encord_agents.tasks import Runner runner = Runner(project_hash="") diff --git a/docs/code_examples/tasks/prioritize_by_data_title_specific.py b/docs/code_examples/tasks/prioritize_by_data_title_specific.py index 2f62b41..5ec80ce 100644 --- a/docs/code_examples/tasks/prioritize_by_data_title_specific.py +++ b/docs/code_examples/tasks/prioritize_by_data_title_specific.py @@ -1,4 +1,5 @@ from encord.objects.ontology_labels_impl import LabelRowV2 + from encord_agents.tasks import Runner runner = Runner(project_hash="") diff --git a/docs/code_examples/tasks/twin_project.py b/docs/code_examples/tasks/twin_project.py index d2fbf54..42b9f46 100644 --- a/docs/code_examples/tasks/twin_project.py +++ b/docs/code_examples/tasks/twin_project.py @@ -1,9 +1,10 @@ from encord.objects.ontology_labels_impl import LabelRowV2 from encord.objects.options import Option from encord.workflow.stages.agent import AgentTask +from typing_extensions import Annotated + from encord_agents.tasks import Depends, Runner from encord_agents.tasks.dependencies import Twin, dep_twin_label_row -from typing_extensions import Annotated # 1. Setup the runner runner = Runner(project_hash="") @@ -19,9 +20,7 @@ @runner.stage(stage="") def copy_labels( manually_annotated_lr: LabelRowV2, - twin: Annotated[ - Twin, Depends(dep_twin_label_row(twin_project_hash="")) - ], + twin: Annotated[Twin, Depends(dep_twin_label_row(twin_project_hash=""))], ) -> str | None: # 4. Reading the checkboxes that have been set instance = manually_annotated_lr.get_classification_instances()[0] diff --git a/docs/code_examples/tasks/wrong_stage_and_pathway_names.py b/docs/code_examples/tasks/wrong_stage_and_pathway_names.py index f5fabcf..5f130df 100644 --- a/docs/code_examples/tasks/wrong_stage_and_pathway_names.py +++ b/docs/code_examples/tasks/wrong_stage_and_pathway_names.py @@ -1,4 +1,5 @@ from encord.objects.ontology_labels_impl import LabelRowV2 + from encord_agents.tasks import Runner r = Runner() diff --git a/encord_agents/core/settings.py b/encord_agents/core/settings.py index ff2cb28..17140b9 100644 --- a/encord_agents/core/settings.py +++ b/encord_agents/core/settings.py @@ -84,4 +84,4 @@ def ssh_key(self) -> str: if self.ssh_key_file is None: raise ValueError("Both ssh key content and ssh key file is None") self.ssh_key_content = self.ssh_key_file.read_text() - return self.ssh_key_content + return self.ssh_key_content diff --git a/encord_agents/core/types.py b/encord_agents/core/types.py index 500f47b..27d113d 100644 --- a/encord_agents/core/types.py +++ b/encord_agents/core/types.py @@ -1,5 +1,3 @@ from typing import Literal Base64Formats = Literal[".jpeg", ".jpg", ".png"] - - diff --git a/encord_agents/core/vision.py b/encord_agents/core/vision.py index 328ebf6..5f86f59 100644 --- a/encord_agents/core/vision.py +++ b/encord_agents/core/vision.py @@ -37,7 +37,7 @@ def rbb_to_poly( [(x + w) * img_width, (y + h) * img_height], [x * img_width, (y + h) * img_height], ], - dtype=np.float32 + dtype=np.float32, ) angle = rbb.theta # [0; 360] center = tuple(bbox_not_rotated.mean(0).tolist()) @@ -110,6 +110,6 @@ def crop_to_object(image: NDArray[np.uint8], coordinates: CroppableCoordinates) return crop_to_bbox(image, box) -def b64_encode_image(img: NDArray[np.uint8], format: Base64Formats =".jpg") -> str: +def b64_encode_image(img: NDArray[np.uint8], format: Base64Formats = ".jpg") -> str: _, encoded_image = cv2.imencode(format, img) return base64.b64encode(encoded_image).decode("utf-8") # type: ignore diff --git a/encord_agents/fastapi/dependencies.py b/encord_agents/fastapi/dependencies.py index 84db5fd..5c147eb 100644 --- a/encord_agents/fastapi/dependencies.py +++ b/encord_agents/fastapi/dependencies.py @@ -147,7 +147,9 @@ def my_route( return get_initialised_label_row(frame_data) -def dep_single_frame(lr: Annotated[LabelRowV2, Depends(dep_label_row)], frame_data: Annotated[FrameData, Form()]) -> NDArray[np.uint8]: +def dep_single_frame( + lr: Annotated[LabelRowV2, Depends(dep_label_row)], frame_data: Annotated[FrameData, Form()] +) -> NDArray[np.uint8]: """ Dependency to inject the underlying asset of the frame data. @@ -214,7 +216,9 @@ def my_route( yield iter_video(asset) -def dep_project(frame_data: Annotated[FrameData, Form()], client: Annotated[EncordUserClient, Depends(dep_client)]) -> Project: +def dep_project( + frame_data: Annotated[FrameData, Form()], client: Annotated[EncordUserClient, Depends(dep_client)] +) -> Project: r""" Dependency to provide an instantiated [Project](https://docs.encord.com/sdk-documentation/sdk-references/LabelRowV2){ target="\_blank", rel="noopener noreferrer" }. diff --git a/encord_agents/tasks/runner.py b/encord_agents/tasks/runner.py index 8f2b042..893168e 100644 --- a/encord_agents/tasks/runner.py +++ b/encord_agents/tasks/runner.py @@ -278,7 +278,7 @@ def _execute_tasks( task.proceed(pathway_name=next_stage, bundle=bundle) if pbar_update is not None: - pbar_update(1.) + pbar_update(1.0) break except KeyboardInterrupt: @@ -474,7 +474,9 @@ def {fn_name}(...): for lr in batch_lrs: if lr: lr.initialise_labels(bundle=lr_bundle) - self._execute_tasks(project, zip(batch, batch_lrs), runner_agent, num_retries, pbar_update=pbar.update) + self._execute_tasks( + project, zip(batch, batch_lrs), runner_agent, num_retries, pbar_update=pbar.update + ) except (PrintableError, AssertionError) as err: if self.was_called_from_cli: panel = Panel(err.args[0], width=None)