Merge pull request #687 from padix-key/ccd

Refactor internal CCD
biotite-dev · Nov 2, 2024 · 72226ca · 72226ca
2 parents 54cd5e5 + cf08dff
commit 72226ca
Show file tree

Hide file tree

Showing 24 changed files with 791 additions and 4,433 deletions.
diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml
@@ -30,6 +30,8 @@ env:
   # Once GHA and cibuildwheel are updated this can be removed
   # mussllinux takes 6+ hrs to build and test so ignore it
   CIBW_TEST_SKIP: "*musllinux* *-macosx_arm64"
+  # Configuration for the architecture-agnostic jobs
+  PY_VERSION: "3.12"  # Keep in sync with version in environment.yml
 
 
 jobs:
@@ -50,6 +52,43 @@ jobs:
       run: ruff check
 
 
+  build-internal:
+    name: Build CCD and wheel for reusing it in several CI jobs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PY_VERSION }}
+      - name: Get current CCD for hashing
+        run: wget https://files.wwpdb.org/pub/pdb/data/monomers/components.cif.gz
+      - name: Cache CCD
+        uses: actions/cache@v4
+        id: cache-ccd
+        with:
+          path: ./src/biotite/structure/info/components.bcif
+          key: cache-${{ hashFiles('setup_ccd.py') }}-${{ hashFiles('components.cif.gz') }}
+      - name: Remove CCD used for hashing
+        run: rm components.cif.gz
+      - name: Build internal CCD
+        if: steps.cache-ccd.outputs.cache-hit != 'true'
+        run: |
+          pip install .
+          python setup_ccd.py
+      - name: Install build backend
+        run: pip install build
+      - name: Build distribution
+        run: python -m build --wheel
+      - uses: actions/upload-artifact@v4
+        with:
+          name: internal-build
+          path: ./dist/*.whl
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ccd
+          path: ./src/biotite/structure/info/components.bcif
+
+
   generate-wheels-matrix:
     name: "Generate wheels matrix"
     runs-on: "ubuntu-latest"
@@ -85,13 +124,20 @@ jobs:
 
   test-and-build:
     name: "Build & Test"
-    needs: "generate-wheels-matrix"
+    needs:
+      - generate-wheels-matrix
+      - build-internal
     strategy:
       matrix:
         include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }}
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - name: Add internal CCD to Biotite
+        uses: actions/download-artifact@v4
+        with:
+          name: ccd
+          path: src/biotite/structure/info
 
       # QEMU enables building/testing for non-native architectures (ie arm64)
       # at the cost of speed
@@ -113,11 +159,18 @@ jobs:
           path: ./wheelhouse/*.whl
 
 
-  make-sdist:
+  sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
+    needs:
+      - build-internal
     steps:
     - uses: actions/checkout@v4
+    - name: Add internal CCD to Biotite
+      uses: actions/download-artifact@v4
+      with:
+        name: ccd
+        path: src/biotite/structure/info
     - name: Build source distribution
       run: pipx run build --sdist
     - uses: actions/upload-artifact@v4
@@ -130,20 +183,24 @@ jobs:
     name: Test interfaces to databases and applications
 
     runs-on: ubuntu-latest
+    needs:
+      - build-internal
     defaults:
       run:
         shell: bash -l {0}
 
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: internal-build
+          path: dist
       - uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: environment.yml
           miniforge-version: latest
-      - name: Build distribution
-        run: pip wheel --no-deps -w dist .
       - name: Install distribution
-        run: pip install .//dist//*.whl
+        run: pip install ./dist/*.whl
       - name: "TEMP: Skip DSSP tests"
         # TEMP: Omit DSSP tests for now until conda-forge DSSP is functional
         # (https://github.com/conda-forge/dssp-feedstock/pull/4)
@@ -161,19 +218,14 @@ jobs:
           tests//test_modname.py
           tests//database
           tests//application
-      # Use the built distribution for other downstream jobs to reduce build time
-      - uses: actions/upload-artifact@v4
-        with:
-          name: internal-build
-          path: ./dist/*.whl
 
 
   test-muscle5:
     name: Test interface to Muscle 5
 
     runs-on: ubuntu-latest
     needs:
-      - test-interfaces
+      - build-internal
     defaults:
       run:
         shell: bash -l {0}
@@ -188,7 +240,7 @@ jobs:
         with:
           activate-environment: biotite-dev
           miniforge-version: latest
-          python-version: "3.12"
+          python-version: ${{ env.PY_VERSION }}
       - name: Install Muscle 5
         run: conda install -c bioconda "muscle=5"
       - name: Install distribution and pytest
@@ -197,46 +249,12 @@ jobs:
         run: pytest --durations=50 tests//application//test_msa.py
 
 
-  make-ccd:
-    name: Compile the CCD subset for structure.info from the wwPDB CCD
-
-    runs-on: ubuntu-latest
-    needs:
-      - test-interfaces
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    steps:
-    - uses: actions/checkout@v4
-    - uses: actions/download-artifact@v4
-      with:
-        name: internal-build
-        path: dist
-    - uses: actions/setup-python@v3
-      with:
-        python-version: "3.12"
-    - name: Install distribution
-      run: pip install dist//*.whl
-    - name: Compile CCD subset
-      run: python setup_ccd.py
-    - name: Zip CCD
-      run: |
-          cd src/biotite/structure/info
-          zip -r ${{ github.workspace }}//dist//ccd.zip ccd
-          cd ${{ github.workspace }}
-    - uses: actions/upload-artifact@v4
-      with:
-        name: ccd
-        path: dist//ccd.zip
-
-
-  make-docs:
+  docs:
     name: Build documentation
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     needs:
-      - test-interfaces
+      - build-internal
     defaults:
       run:
         shell: bash -l {0}
@@ -258,7 +276,7 @@ jobs:
         environment-file: environment.yml
         miniforge-version: latest
     - name: Install distribution
-      run: pip install dist//*.whl
+      run: pip install dist/*.whl
     - name: Build base documentation
       run: sphinx-build -a -D plot_gallery=0 doc build//doc
     - name: Build tutorial and gallery
@@ -285,16 +303,16 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name != 'release'
     needs:
-      - test-interfaces
+      - build-internal
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
           name: internal-build
           path: dist
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ env.PY_VERSION }}
       - name: Install dependencies
         run: pip install dist//*.whl pytest pytest-codspeed
       - name: Run benchmarks
@@ -310,7 +328,7 @@ jobs:
     needs:
     - lint
     - test-and-build
-    - make-sdist
+    - sdist
     - test-interfaces
     - test-muscle5
     runs-on: ubuntu-latest
@@ -334,12 +352,31 @@ jobs:
         password: ${{ secrets.PYPI_TOKEN }}
 
 
+  upload-ccd:
+    name: Upload CCD to GitHub Releases
+    permissions:
+      contents: write
+    needs:
+    - build-internal
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/download-artifact@v4
+      with:
+        name: ccd
+        path: dist
+    - name: Upload to GitHub Releases
+      uses: softprops/[email protected]
+      if: github.event_name == 'release' && github.event.action == 'published'
+      with:
+        files: dist//components.bcif
+
+
   upload-docs:
     name: Upload documentation to GitHub Releases
     permissions:
       contents: write
     needs:
-    - make-docs
+    - docs
     runs-on: ubuntu-latest
     steps:
     - uses: actions/download-artifact@v4

diff --git a/.gitignore b/.gitignore
@@ -20,6 +20,9 @@ htmlcov
 # Ignore version file created by hatch-vcs
 /src/biotite/version.py
 
+# Ignore internal CCD
+/src/biotite/structure/info/components.bcif
+
 # Ignore autogenerated documentation files
 /doc/static/switcher.json
 /doc/apidoc

diff --git a/doc/apidoc.json b/doc/apidoc.json
@@ -345,9 +345,29 @@
             "base_pairs_from_dot_bracket"
         ]
     },
-
+    "biotite.structure.info" : {
+        "Residues and bonds": [
+            "residue",
+            "bond_type",
+            "bonds_in_residue"
+        ],
+        "Component groups": [
+            "amino_acid_names",
+            "nucleotide_names",
+            "carbohydrate_names"
+        ],
+        "Atom radii": [
+            "vdw_radius_single",
+            "vdw_radius_protor"
+        ],
+        "Low-level CCD access": [
+            "get_ccd",
+            "get_from_ccd",
+            "set_ccd_path"
+        ]
+    },
     "biotite.structure.io.pdbx" : {
-        "High-level functionality": [
+        "High-level functionality" : [
             "get_sequence",
             "get_model_count",
             "get_structure",

diff --git a/doc/install.rst b/doc/install.rst
@@ -53,38 +53,35 @@ Installation via Conda
 Installation from source
 ------------------------
 
-You can also install Biotite from the
+You can also install *Biotite* from the
 `project repository <https://github.com/biotite-dev/biotite>`_.
+However, in addition to building and installing the package, the internal
+`Chemical Component Dictionary (CCD) <https://www.wwpdb.org/data/ccd>`_. for
+:mod:`biotite.structure.info` needs to be built with the ``setup_ccd.py`` script.
+The script in turn requires *Biotite*.
+The solution to this chicken-and-egg problem is to first install Biotite without the
+CCD, then build the CCD and finally install Biotite again.
 After cloning the repository, navigate to its top-level directory (the one
 ``setup.py`` is in) and type the following:
 
 .. code-block:: console
 
    $ pip install .
+   $ python setup_ccd.py
+   $ pip install .
+
+The `setup_ccd.py` script can also be used to update the internal CCD to the current
+upstream version from the PDB.
 
 Having the *Biotite* installation always pointing to your repository clone is
 also possible.
-Type the following in the top-level directory:
+Substitute the installation with the following commands instead:
 
 .. code-block:: console
 
    $ pip install -e .
-
-Updating the Chemical Component Dictionary
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The :mod:`biotite.structure.info` subpackage contains a subset from the
-`PDB Chemical Component Dictionary (CCD) <https://www.wwpdb.org/data/ccd>`_.
-The repository ships a potentially outdated version of this subset.
-To update this subset to the current upstream CCD version, run
-
-.. code-block:: console
-
    $ python setup_ccd.py
 
-Afterwards, install *Biotite* again.
-
-
 Common issues and solutions
 ---------------------------
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -102,6 +102,7 @@ filterwarnings = [
   "ignore:Input structure has no associated 'BondList'",
 ]
 
+
 [tool.hatch.build.targets.sdist]
 exclude = [
   "tests",
@@ -112,6 +113,14 @@ exclude = [
   # .github, .gitignore, .gitattributes
   ".git*",
 ]
+artifacts = [
+  "src/biotite/structure/info/components.bcif"
+]
+
+[tool.hatch.build.targets.wheel]
+artifacts = [
+  "src/biotite/structure/info/components.bcif"
+]
 
 [tool.hatch.version]
 source = "vcs"