diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4df1860cf..f9a4798e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,25 +17,39 @@ jobs: python-version: - "3.10" - "3.11" + - "3.12" fail-fast: false steps: - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 + id: setup_python with: python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: requirements*.txt - - name: Install dependencies + - name: UV Cache + # Manually cache the uv cache directory + # until setup-python supports it: + # https://github.com/actions/setup-python/issues/822 + uses: actions/cache@v4 + id: cache-uv + with: + path: ~/.cache/uv + key: uvcache-${{ runner.os }}-${{ hashFiles('requirements.txt') }} + + - name: Install packages run: | - python -m pip install --upgrade pip - python -m pip install --upgrade --upgrade-strategy eager -e ".[dev]" + python -m pip install -U uv + uv pip install --upgrade --system -e .[dev] - name: Run tests env: PREFECT_SERVER_DATABASE_CONNECTION_URL: "sqlite+aiosqlite:///./collection-tests.db" run: | - coverage run --branch -m pytest tests -vv - coverage report \ No newline at end of file + pytest -vv \ No newline at end of file diff --git a/deployment.yaml b/deployment.yaml deleted file mode 100644 index df1642e23..000000000 --- a/deployment.yaml +++ /dev/null @@ -1,37 +0,0 @@ -definitions: - work_pools: - kubernetes_prd_internal_tools: &kubernetes_prd_internal_tools - name: kubernetes-prd-internal-tools - job_variables: - image: prefecthq/prefect:2-python3.11 - env: - EXTRA_PIP_PACKAGES: github3.py fastjsonschema gcsfs - -deployments: - - name: update-a-collection - version: null - tags: [] - description: Updates each variety of metadata for a given package. - schedule: {} - flow_name: null - entrypoint: src/update_collection_metadata.py:update_collection_metadata - parameters: {} - work_pool: *kubernetes_prd_internal_tools - - - name: update-all-collections - version: null - tags: [] - description: |- - `update-all-collections` triggers many instances of `update-collection-metadata` - in order to update the [prefect-collection-registry](https://github.com/PrefectHQ/prefect-collection-registry) - with metadata generated from new releases of select packages (prefect collections + prefect core). - - `update-all-collections` flow will check if any packages have a release and are not - recorded by the registry repo, and will trigger a run of `update_collection_metadata` for each such package. - schedule: - cron: 0 6,12,18 * * 1-5 - timezone: EST - flow_name: null - entrypoint: src/update_collection_metadata.py:update_all_collections - parameters: {} - work_pool: *kubernetes_prd_internal_tools diff --git a/prefect.yaml b/prefect.yaml index 8432f6ed4..c69b170f0 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -1,17 +1,52 @@ -# File for configuring project / deployment build, push and pull steps - -# Generic metadata about this project name: prefect-collection-registry -prefect-version: 2.10.6 +prefect-version: 3.0.0 + + +definitions: + work_pools: + kubernetes_prd_internal_tools: &kubernetes_prd_internal_tools + name: kubernetes-prd-internal-tools + job_variables: + image: prefecthq/prefect:3.0.0-python3.12 + env: + EXTRA_PIP_PACKAGES: github3.py fastjsonschema prefect-gcp + -# build section allows you to manage and build docker images -build: null +build: -# push section allows you to manage if and how this project is uploaded to remote locations -push: null +push: -# pull section allows you to provide instructions for cloning this project in remote locations pull: - - prefect.deployments.steps.git_clone: - repository: https://github.com/PrefectHQ/prefect-collection-registry - branch: main +- prefect.deployments.steps.git_clone: + repository: https://github.com/PrefectHQ/prefect-collection-registry + branch: main + + +deployments: +- name: update-a-collection + tags: [] + description: Updates each variety of metadata for a given package. + schedules: + entrypoint: src/update_collection_metadata.py:update_collection_metadata + parameters: {} + work_pool: *kubernetes_prd_internal_tools + +- name: update-all-collections + tags: [] + description: "`update-all-collections` triggers many instances of `update-collection-metadata` + \nin order to update the [prefect-collection-registry](https://github.com/PrefectHQ/prefect-collection-registry) + \nwith metadata generated from new releases of select packages (prefect collections + + prefect core).\n\n`update-all-collections` flow will check if any packages have + a release and are not \nrecorded by the registry repo, and will trigger a run + of `update_collection_metadata` for each such package." + schedules: + - cron: 0 6,12,18 * * 1-5 + timezone: EST + day_or: true + active: true + max_active_runs: + catchup: false + entrypoint: src/update_collection_metadata.py:update_all_collections + work_pool: *kubernetes_prd_internal_tools + parameters: {} + enforce_parameter_schema: true diff --git a/pyproject.toml b/pyproject.toml index e43f5d354..583c04036 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,21 +4,12 @@ build-backend = "setuptools.build_meta" [project] name = "prefect-collection-registry" -authors = [ - {name = "Prefect Technologies", email = "help@prefect.io"}, -] +authors = [{ name = "Prefect Technologies", email = "help@prefect.io" }] description = "Maintaining the Prefect Collection Registry" readme = "README.md" requires-python = ">=3.10" -classifiers = [ - "Programming Language :: Python :: 3", -] -dependencies = [ - "prefect>=2.13.5", - "fastjsonschema==2.16.2", - "github3.py>=3.2.0", - -] +classifiers = ["Programming Language :: Python :: 3"] +dependencies = ["prefect>=3.0.0", "fastjsonschema==2.16.2", "github3.py>=3.2.0"] dynamic = ["version"] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index 7c279eefd..935e5eb7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ fastjsonschema gcsfs github3.py==3.2.0 -prefect>=2.10.6 +prefect>=3.0.0 diff --git a/src/generate_flow_metadata.py b/src/generate_flow_metadata.py index 309acf1e4..0fca3fa62 100644 --- a/src/generate_flow_metadata.py +++ b/src/generate_flow_metadata.py @@ -2,9 +2,7 @@ from typing import Any, Callable, Dict, List import fastjsonschema -from griffe.dataclasses import Docstring -from griffe.docstrings.dataclasses import DocstringSectionKind -from griffe.docstrings.parsers import Parser, parse +from griffe import Docstring, DocstringSectionKind, Parser, parse from prefect import Flow, flow, task from prefect.states import Completed from prefect.utilities.importtools import load_module diff --git a/src/update_collection_metadata.py b/src/update_collection_metadata.py index 2f7f31245..b108be80f 100644 --- a/src/update_collection_metadata.py +++ b/src/update_collection_metadata.py @@ -3,16 +3,17 @@ import github3 import pendulum -import utils -from generate_block_metadata import update_block_metadata_for_collection -from generate_flow_metadata import update_flow_metadata_for_collection -from generate_worker_metadata import update_worker_metadata_for_package from prefect import flow, task +from prefect.client.schemas.objects import FlowRun from prefect.deployments import run_deployment -from prefect.server.schemas.core import FlowRun from prefect.states import Completed, Failed, State from prefect.utilities.collections import listrepr +import utils +from generate_block_metadata import update_block_metadata_for_collection +from generate_flow_metadata import update_flow_metadata_for_collection +from generate_worker_metadata import update_worker_metadata_for_package + UPDATE_ALL_DESCRIPTION = """ The `update_all_collections` triggers many instances of `update_collection_metadata` in order to update the [`prefect-collection-registry`](https://github.com/PrefectHQ/prefect-collection-registry) @@ -147,12 +148,12 @@ def update_collection_metadata( @flow( - description=UPDATE_ALL_DESCRIPTION, - log_prints=True, name="update-all-collections", - result_storage=utils.result_storage_from_env(), + description=UPDATE_ALL_DESCRIPTION, + result_storage="gcs-bucket/collection-registry-storage", retries=2, retry_delay_seconds=10, + log_prints=True, ) async def update_all_collections( branch_name: str = "update-metadata", @@ -199,9 +200,9 @@ async def update_all_collections( return Completed(message="All new releases have been recorded.") -# if __name__ == "__main__": -# # ALL COLLECTIONS -# asyncio.run(update_all_collections()) +if __name__ == "__main__": + # ALL COLLECTIONS + asyncio.run(update_all_collections()) # # MANUAL RUNS # for collection in ["prefect-sqlalchemy"]: diff --git a/src/utils.py b/src/utils.py index 73e86ffe0..8ef4e904a 100644 --- a/src/utils.py +++ b/src/utils.py @@ -10,10 +10,7 @@ import httpx import yaml from prefect import Flow, task -from prefect.blocks.core import Block from prefect.blocks.system import Secret -from prefect.client.cloud import get_cloud_client -from prefect.settings import PREFECT_API_URL from prefect.utilities.asyncutils import ( run_sync_in_worker_thread, # noqa sync_compatible, @@ -270,21 +267,3 @@ def validate_view_content(view_dict: dict, variety: CollectionViewVariety) -> No except IndexError: # to catch something like {"prefect-X": {}} raise ValueError("There's a key with empty value in this view!") print(f" Validated {collection_name} summary in {variety} view!") - - -@sync_compatible -async def result_storage_from_env() -> Block | None: - env_to_storage_block_name = { - "inconspicuous-pond": "s3/flow-script-storage", - "integrations": "gcs/collection-registry-result-storage", - } - - async with get_cloud_client() as client: - current_workspace_id = PREFECT_API_URL.value().split("/")[-1] - - for workspace in await client.read_workspaces(): - if str(workspace.workspace_id) == current_workspace_id: - result_storage_name = env_to_storage_block_name[ - workspace.workspace_name - ] - return await Block.load(name=result_storage_name) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..19d551b83 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,8 @@ +import pytest +from prefect.testing.utilities import prefect_test_harness + + +@pytest.fixture(autouse=True) +def prefect_db(): + with prefect_test_harness(): + yield