Statically bundle dandi/schema json

.github/workflows/release.yml

@@ -47,6 +47,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          submodules: true
           path: dandischema
           token: ${{ secrets.DANDI_GITHUB_TOKEN }}
 
@@ -73,35 +74,50 @@ jobs:
           fi
           echo "SCHEMA_VERSION=$SCHEMA_VERSION" >> "$GITHUB_ENV"
 
-      - name: Checkout dandi/schema
-        uses: actions/checkout@v3
-        with:
-          repository: dandi/schema
-          path: schema
-          token: ${{ secrets.DANDI_GITHUB_TOKEN }}
-
       - name: Test for unversioned changes
         run: |
           dirs="$(echo releases/*)"
           echo "Following directories will be checked for differences: $dirs"
-          python ../dandischema/tools/pubschemata.py releases
+          python ../../tools/pubschemata.py releases
           git status
           git add -A releases
           if ! git diff --cached --exit-code $dirs
           then echo "[ERROR] Existing schema files modified instead of creating a new version"
                exit 1
           fi
-        working-directory: schema
-
-      - name: Install build & twine
-        run: python -m pip install build twine
+        working-directory: dandischema/dandischema/schema
 
       - name: Download auto
         run: |
           auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/tags/$AUTO_VERSION | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')"
           wget -O- "$auto_download_url" | gunzip > ~/auto
           chmod a+x ~/auto
 
+      - name: Commit schema changes
+        run: |
+          LIBRARY_VERSION="$(auto shipit --dry-run --quiet)"
+
+          git config --global user.email "[email protected]"
+          git config --global user.name "DANDI Bot"
+
+          git add releases
+          if ! git diff --quiet --cached
+          then git commit -m "Publish model schema v$SCHEMA_VERSION as of dandischema v$LIBRARY_VERSION"
+               git push
+          else echo "No changes to commit"
+          fi
+        working-directory: dandischema/dandischema/schema
+
+      - name: Update schema submodule with new release
+        run: |
+          git submodule update
+          git add dandischema/schema
+          git commit -m "Update to include schema v$SCHEMA_VERSION"
+        working-directory: dandischema
+
+      - name: Install build & twine
+        run: python -m pip install build twine
+
       - name: Create release
         run: |
           ~/auto shipit -vv
@@ -113,21 +129,10 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
         working-directory: dandischema
 
-      - name: Commit schema changes and create new tag
+      - name: Add schema tag to release commit
         run: |
-          LIBRARY_VERSION="$(git -C ../dandischema describe --tags --exact-match)"
-
-          git config --global user.email "[email protected]"
-          git config --global user.name "DANDI Bot"
-
-          git add releases
-          if ! git diff --quiet --cached
-          then git commit -m "Publish model schema v$SCHEMA_VERSION as of dandischema v$LIBRARY_VERSION"
-               git push
-               git -C ../dandischema tag -m "Schema v$SCHEMA_VERSION, released in dandischema v$LIBRARY_VERSION" schema-$SCHEMA_VERSION
-               git -C ../dandischema push --tags
-          else echo "No changes to commit"
-          fi
-        working-directory: schema
+          git tag -m "Update version to $SCHEMA_VERSION" schema-$SCHEMA_VERSION
+          git push --tags
+        working-directory: dandischema
 
 # vim:set sts=2:

.github/workflows/test.yml

@@ -31,6 +31,7 @@ jobs:
         # Fetch all commits so that versioningit will return something
         # compatible with semantic-version
         fetch-depth: 0
+        submodules: true
 
     - name: Set up Python ${{ matrix.python }}
       uses: actions/setup-python@v4

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "schema"]
+    path = dandischema/schema
+    url = [email protected]:dandi/schema.git

MANIFEST.in

@@ -1,3 +1,5 @@
 include CHANGELOG.md tox.ini
 graft dandischema
+prune dandischema/schema
+recursive-include dandischema/schema/releases *.json
 global-exclude *.py[cod]

dandischema/metadata.py

@@ -1,11 +1,17 @@
 from copy import deepcopy
 import json
 from pathlib import Path
+import sys
 from typing import Any, Dict, Iterable, TypeVar, cast
 
+if sys.version_info >= (3, 10, 0):
+    # Available in python >= 3.10
+    from importlib.resources import files
+else:
+    from importlib_resources import files
+
 import jsonschema
 import pydantic
-import requests
 
 from .consts import (
     ALLOWED_INPUT_SCHEMAS,
@@ -129,6 +135,14 @@ def _validate_asset_json(data: dict, schema_dir: str) -> None:
     _validate_obj_json(data, schema)
 
 
+def _get_schema(schema_version: str, schema_name: str) -> dict:
+    return json.loads(
+        files("dandischema.schema")
+        .joinpath(f"releases/{schema_version}/{schema_name}")
+        .read_text()
+    )
+
+
 def validate(
     obj, schema_version=None, schema_key=None, missing_ok=False, json_validation=False
 ):
@@ -181,10 +195,7 @@ def validate(
                     "using json schema for older versions"
                 )
             schema_filename = schema_map[schema_key]
-            schema = requests.get(
-                f"https://raw.githubusercontent.com/dandi/schema/"
-                f"master/releases/{schema_version}/{schema_filename}"
-            ).json()
+            schema = _get_schema(schema_version, schema_filename)
         _validate_obj_json(obj, schema, missing_ok)
     klass = getattr(models, schema_key)
     try:
@@ -217,10 +228,7 @@ def migrate(
     if version2tuple(schema_version) > version2tuple(to_version):
         raise ValueError(f"Cannot migrate from {schema_version} to lower {to_version}.")
     if not (skip_validation):
-        schema = requests.get(
-            f"https://raw.githubusercontent.com/dandi/schema/"
-            f"master/releases/{schema_version}/dandiset.json"
-        ).json()
+        schema = _get_schema(schema_version, "dandiset.json")
         _validate_obj_json(obj, schema)
     if version2tuple(schema_version) < version2tuple("0.6.0"):
         for val in obj.get("about", []):

setup.cfg

@@ -28,6 +28,9 @@ project_urls =
 [options]
 python_requires = >=3.7
 install_requires =
+    # importlib-resources can be removed once 3.10 becomes the minimum required version.
+    # https://setuptools.pypa.io/en/latest/userguide/datafiles.html#accessing-data-files-at-runtime
+    importlib-resources; python_version < "3.10"
     jsonschema[format]
     pydantic[email] >= 1.8.1
     typing_extensions;  python_version < "3.8"