Merge pull request #419 from PrefectHQ/3.0

upgrade to 3

.github/workflows/tests.yml

@@ -17,25 +17,39 @@ jobs:
         python-version:
           - "3.10"
           - "3.11"
+          - "3.12"
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
 
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
+        id: setup_python
         with:
           python-version: ${{ matrix.python-version }}
-          cache: pip
-          cache-dependency-path: requirements*.txt
 
-      - name: Install dependencies
+      - name: UV Cache
+        # Manually cache the uv cache directory
+        # until setup-python supports it:
+        # https://github.com/actions/setup-python/issues/822
+        uses: actions/cache@v4
+        id: cache-uv
+        with:
+          path: ~/.cache/uv
+          key: uvcache-${{ runner.os }}-${{ hashFiles('requirements.txt') }}
+
+      - name: Install packages
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade --upgrade-strategy eager -e ".[dev]"
+          python -m pip install -U uv
+          uv pip install --upgrade --system -e .[dev]
 
       - name: Run tests
         env:
           PREFECT_SERVER_DATABASE_CONNECTION_URL: "sqlite+aiosqlite:///./collection-tests.db"
         run: |
-          coverage run --branch -m pytest tests -vv
-          coverage report
+          pytest -vv

prefect.yaml

@@ -1,17 +1,52 @@
-# File for configuring project / deployment build, push and pull steps
-
-# Generic metadata about this project
 name: prefect-collection-registry
-prefect-version: 2.10.6
+prefect-version: 3.0.0
+
+
+definitions:
+  work_pools:
+    kubernetes_prd_internal_tools: &kubernetes_prd_internal_tools
+      name: kubernetes-prd-internal-tools
+      job_variables:
+        image: prefecthq/prefect:3.0.0-python3.12
+        env:
+          EXTRA_PIP_PACKAGES: github3.py fastjsonschema prefect-gcp
+
 
-# build section allows you to manage and build docker images
-build: null
+build:
 
-# push section allows you to manage if and how this project is uploaded to remote locations
-push: null
+push:
 
-# pull section allows you to provide instructions for cloning this project in remote locations
 pull:
-  - prefect.deployments.steps.git_clone:
-      repository: https://github.com/PrefectHQ/prefect-collection-registry
-      branch: main
+- prefect.deployments.steps.git_clone:
+    repository: https://github.com/PrefectHQ/prefect-collection-registry
+    branch: main
+
+
+deployments:
+- name: update-a-collection
+  tags: []
+  description: Updates each variety of metadata for a given package.
+  schedules:
+  entrypoint: src/update_collection_metadata.py:update_collection_metadata
+  parameters: {}
+  work_pool: *kubernetes_prd_internal_tools
+
+- name: update-all-collections
+  tags: []
+  description: "`update-all-collections` triggers many instances of `update-collection-metadata`
+    \nin order to update the [prefect-collection-registry](https://github.com/PrefectHQ/prefect-collection-registry)
+    \nwith metadata generated from new releases of select packages (prefect collections
+    + prefect core).\n\n`update-all-collections` flow will check if any packages have
+    a release and are not \nrecorded by the registry repo, and will trigger a run
+    of `update_collection_metadata` for each such package."
+  schedules:
+  - cron: 0 6,12,18 * * 1-5
+    timezone: EST
+    day_or: true
+    active: true
+    max_active_runs:
+    catchup: false
+  entrypoint: src/update_collection_metadata.py:update_all_collections
+  work_pool: *kubernetes_prd_internal_tools
+  parameters: {}
+  enforce_parameter_schema: true

pyproject.toml

@@ -4,21 +4,12 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "prefect-collection-registry"
-authors = [
-    {name = "Prefect Technologies", email = "[email protected]"},
-]
+authors = [{ name = "Prefect Technologies", email = "[email protected]" }]
 description = "Maintaining the Prefect Collection Registry"
 readme = "README.md"
 requires-python = ">=3.10"
-classifiers = [
-    "Programming Language :: Python :: 3",
-]
-dependencies = [
-    "prefect>=2.13.5",
-    "fastjsonschema==2.16.2",
-    "github3.py>=3.2.0",
-
-]
+classifiers = ["Programming Language :: Python :: 3"]
+dependencies = ["prefect>=3.0.0", "fastjsonschema==2.16.2", "github3.py>=3.2.0"]
 dynamic = ["version"]
 
 [project.optional-dependencies]

requirements.txt

@@ -1,4 +1,4 @@
 fastjsonschema
 gcsfs
 github3.py==3.2.0
-prefect>=2.10.6
+prefect>=3.0.0

src/generate_flow_metadata.py

@@ -2,9 +2,7 @@
 from typing import Any, Callable, Dict, List
 
 import fastjsonschema
-from griffe.dataclasses import Docstring
-from griffe.docstrings.dataclasses import DocstringSectionKind
-from griffe.docstrings.parsers import Parser, parse
+from griffe import Docstring, DocstringSectionKind, Parser, parse
 from prefect import Flow, flow, task
 from prefect.states import Completed
 from prefect.utilities.importtools import load_module

src/update_collection_metadata.py

@@ -3,16 +3,17 @@
 
 import github3
 import pendulum
-import utils
-from generate_block_metadata import update_block_metadata_for_collection
-from generate_flow_metadata import update_flow_metadata_for_collection
-from generate_worker_metadata import update_worker_metadata_for_package
 from prefect import flow, task
+from prefect.client.schemas.objects import FlowRun
 from prefect.deployments import run_deployment
-from prefect.server.schemas.core import FlowRun
 from prefect.states import Completed, Failed, State
 from prefect.utilities.collections import listrepr
 
+import utils
+from generate_block_metadata import update_block_metadata_for_collection
+from generate_flow_metadata import update_flow_metadata_for_collection
+from generate_worker_metadata import update_worker_metadata_for_package
+
 UPDATE_ALL_DESCRIPTION = """
 The `update_all_collections` triggers many instances of `update_collection_metadata` in order to
 update the [`prefect-collection-registry`](https://github.com/PrefectHQ/prefect-collection-registry)
@@ -147,12 +148,12 @@ def update_collection_metadata(
 
 
 @flow(
-    description=UPDATE_ALL_DESCRIPTION,
-    log_prints=True,
     name="update-all-collections",
-    result_storage=utils.result_storage_from_env(),
+    description=UPDATE_ALL_DESCRIPTION,
+    result_storage="gcs-bucket/collection-registry-storage",
     retries=2,
     retry_delay_seconds=10,
+    log_prints=True,
 )
 async def update_all_collections(
     branch_name: str = "update-metadata",
@@ -199,9 +200,9 @@ async def update_all_collections(
     return Completed(message="All new releases have been recorded.")
 
 
-# if __name__ == "__main__":
-# # ALL COLLECTIONS
-# asyncio.run(update_all_collections())
+if __name__ == "__main__":
+    # ALL COLLECTIONS
+    asyncio.run(update_all_collections())
 
 # # MANUAL RUNS
 # for collection in ["prefect-sqlalchemy"]:

src/utils.py

@@ -10,10 +10,7 @@
 import httpx
 import yaml
 from prefect import Flow, task
-from prefect.blocks.core import Block
 from prefect.blocks.system import Secret
-from prefect.client.cloud import get_cloud_client
-from prefect.settings import PREFECT_API_URL
 from prefect.utilities.asyncutils import (
     run_sync_in_worker_thread,  # noqa
     sync_compatible,
@@ -270,21 +267,3 @@ def validate_view_content(view_dict: dict, variety: CollectionViewVariety) -> No
         except IndexError:  # to catch something like {"prefect-X": {}}
             raise ValueError("There's a key with empty value in this view!")
         print(f"  Validated {collection_name} summary in {variety} view!")
-
-
-@sync_compatible
-async def result_storage_from_env() -> Block | None:
-    env_to_storage_block_name = {
-        "inconspicuous-pond": "s3/flow-script-storage",
-        "integrations": "gcs/collection-registry-result-storage",
-    }
-
-    async with get_cloud_client() as client:
-        current_workspace_id = PREFECT_API_URL.value().split("/")[-1]
-
-        for workspace in await client.read_workspaces():
-            if str(workspace.workspace_id) == current_workspace_id:
-                result_storage_name = env_to_storage_block_name[
-                    workspace.workspace_name
-                ]
-                return await Block.load(name=result_storage_name)

tests/conftest.py

@@ -0,0 +1,8 @@
+import pytest
+from prefect.testing.utilities import prefect_test_harness
+
+
+@pytest.fixture(autouse=True)
+def prefect_db():
+    with prefect_test_harness():
+        yield