Merge pull request #100 from inab/full_circle

Full circle reusing payloads (alpha quality) reached!

.github/workflows/pre-commit.yml

@@ -43,7 +43,7 @@ jobs:
           path: .mypy_cache/${{ matrix.python-version }}
           key: mypy-${{ matrix.python-version }}
       - name: 'pre-commit'
-        uses: pre-commit/[email protected].0
+        uses: pre-commit/[email protected].1
 #        if: ${{ matrix.python-version != '3.6' }}
         with:
           extra_args: --all -c .pre-commit-config.yaml
@@ -70,6 +70,8 @@ jobs:
           path: constraints-${{ matrix.python-version }}.txt
 
   pull_request_changes:
+    # Do this only when it is not a pull request validation
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     name: Pull request with the newly generated contents
     needs:

.github/workflows/schemas-doc-generator.yml

@@ -1,7 +1,6 @@
 name: schemas-doc-generator
 
 on:
-  pull_request:
   push:
     branches: [main]
 
@@ -16,6 +15,8 @@ jobs:
           extra_args: --all -c .pre-commit-config.yaml jsonschema_dir_validate
 
   update_docs:
+    # Do this only when it is not a pull request validation
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
     name: Update documentation if all worked properly
     needs:

CITATION.cff

@@ -28,4 +28,4 @@ message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/inab/WfExS-backend"
 type: software
 title: "WfExS-backend"
-version: 0.99.2
+version: 0.99.9

requirements.txt

@@ -1,14 +1,16 @@
 pyyaml
 jsonpath-ng
 jsonschema[format_nongpl] >= 3.0.0
-jsonschema[format_nongpl] < 4.18.0 ; python_version <= '3.7'
+jsonschema[format_nongpl] < 4.18.0 ; python_version < '3.8'
 jsonpointer
 python-magic >= 0.4.27
 paramiko[ed25519]
 certifi
 crypt4gh
 bagit
 aioftp
+aiohttp-client-cache[sqlite] >= 0.11.0 ; python_version >= '3.8'
+aiohttp-client-cache[sqlite] < 0.11.0 ; python_version < '3.8'
 openpyxl
 xlrd2
 dulwich

tests/containers/test_docker.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import logging
+
+from typing import (
+    cast,
+    TYPE_CHECKING,
+)
+
+if TYPE_CHECKING:
+    from wfexs_backend.common import (
+        Fingerprint,
+        RelPath,
+        URIType,
+    )
+
+    from wfexs_backend.container_factories import (
+        ContainerOperatingSystem,
+        ProcessorArchitecture,
+    )
+
+from wfexs_backend.common import (
+    ContainerTaggedName,
+    ContainerType,
+)
+
+from wfexs_backend.container_factories import (
+    Container,
+    ContainerEngineException,
+    ContainerFactoryException,
+)
+
+from wfexs_backend.container_factories.docker_container import (
+    DockerContainerFactory,
+)
+
+from tests.containers.util import simpleTestContainerFileName
+
+# Enable logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def test_docker_basic(tmpdir) -> "None":  # type: ignore[no-untyped-def]
+    """
+    Check docker container factory instantiation
+    """
+    temppath = tmpdir.mkdir("TEMP")
+    cachepath = tmpdir.mkdir("CACHE")
+    stagedpath = tmpdir.mkdir("STAGED")
+    dcf = DockerContainerFactory(
+        simpleFileNameMethod=simpleTestContainerFileName,
+        containersCacheDir=cachepath.strpath,
+        stagedContainersDir=stagedpath.strpath,
+        tempDir=temppath.strpath,
+    )
+
+
+TAGGED_TESTBED = pytest.mark.parametrize(
+    ["cont_tagged"],
+    [
+        (
+            ContainerTaggedName(
+                origTaggedName="busybox:stable",
+                type=ContainerType.Docker,
+            ),
+        ),
+        (
+            ContainerTaggedName(
+                origTaggedName="quay/busybox:latest",
+                type=ContainerType.Docker,
+                registries={
+                    ContainerType.Docker: "quay.io",
+                },
+            ),
+        ),
+        (
+            Container(
+                origTaggedName="busybox:stable",
+                type=ContainerType.Docker,
+                taggedName=cast("URIType", "busybox:stable"),
+                operatingSystem=cast("ContainerOperatingSystem", "linux"),
+                architecture=cast("ProcessorArchitecture", "amd64"),
+                signature=cast(
+                    "Fingerprint", "sha256=sBYlsI2WjxCwGm9juxawq1ryW3OpivFxFUWxEvQ9vBU="
+                ),
+                fingerprint=cast(
+                    "Fingerprint",
+                    "docker.io/library/busybox@sha256:9ae97d36d26566ff84e8893c64a6dc4fe8ca6d1144bf5b87b2b85a32def253c7",
+                ),
+                source_type=ContainerType.Docker,
+                image_signature=cast(
+                    "Fingerprint", "sha256=sBYlsI2WjxCwGm9juxawq1ryW3OpivFxFUWxEvQ9vBU="
+                ),
+            ),
+        ),
+        (
+            Container(
+                origTaggedName="quay/busybox:latest",
+                type=ContainerType.Docker,
+                registries={
+                    ContainerType.Docker: "quay.io",
+                },
+                taggedName=cast("URIType", "quay.io/quay/busybox:latest"),
+                operatingSystem=cast("ContainerOperatingSystem", "linux"),
+                architecture=cast("ProcessorArchitecture", "amd64"),
+                signature=cast(
+                    "Fingerprint", "sha256=WTkWLbkE2f3HvwpLcWIOMaW85YxuZBCPXSffBez6hKY="
+                ),
+                fingerprint=cast(
+                    "Fingerprint",
+                    "quay.io/quay/busybox@sha256:92f3298bf80a1ba949140d77987f5de081f010337880cd771f7e7fc928f8c74d",
+                ),
+                source_type=ContainerType.Docker,
+                image_signature=cast(
+                    "Fingerprint", "sha256=WTkWLbkE2f3HvwpLcWIOMaW85YxuZBCPXSffBez6hKY="
+                ),
+            ),
+        ),
+    ],
+)
+
+
+@TAGGED_TESTBED
+def test_docker_container_tagged_name(cont_tagged: "ContainerTaggedName", tmpdir) -> "None":  # type: ignore[no-untyped-def]
+    """
+    Check docker container factory instantiation
+    """
+    temppath = tmpdir.mkdir("TEMP")
+    cachepath = tmpdir.mkdir("CACHE")
+    stagedpath = tmpdir.mkdir("STAGED")
+    dcf = DockerContainerFactory(
+        simpleFileNameMethod=simpleTestContainerFileName,
+        containersCacheDir=cachepath.strpath,
+        stagedContainersDir=stagedpath.strpath,
+        tempDir=temppath.strpath,
+    )
+    containers = dcf.materializeContainers(tagList=[cont_tagged])
+    if isinstance(cont_tagged, Container):
+        for attr in (
+            "origTaggedName",
+            "type",
+            "registries",
+            "taggedName",
+            "architecture",
+            "operatingSystem",
+            "fingerprint",
+            "source_type",
+        ):
+            assert getattr(cont_tagged, attr) == getattr(
+                containers[0], attr
+            ), f"Expected and obtainer container '{attr}' do not match: {getattr(cont_tagged, attr)} vs {getattr(containers[0], attr)}"
+
+
+@TAGGED_TESTBED
+def test_docker_container_tagged_name_fail(cont_tagged: "ContainerTaggedName", tmpdir) -> "None":  # type: ignore[no-untyped-def]
+    """
+    Check docker container factory instantiation
+    """
+    temppath = tmpdir.mkdir("TEMP")
+    cachepath = tmpdir.mkdir("CACHE")
+    stagedpath = tmpdir.mkdir("STAGED")
+    dcf = DockerContainerFactory(
+        simpleFileNameMethod=simpleTestContainerFileName,
+        containersCacheDir=cachepath.strpath,
+        stagedContainersDir=stagedpath.strpath,
+        tempDir=temppath.strpath,
+    )
+    with pytest.raises(ContainerFactoryException):
+        containers = dcf.materializeContainers(tagList=[cont_tagged], offline=True)
+        logger.info(containers)
+
+
+@TAGGED_TESTBED
+def test_docker_container_tagged_name_cached(cont_tagged: "ContainerTaggedName", tmpdir) -> "None":  # type: ignore[no-untyped-def]
+    """
+    Check docker container factory instantiation
+    """
+    temppath = tmpdir.mkdir("TEMP")
+    cachepath = tmpdir.mkdir("CACHE")
+    stagedpath = tmpdir.mkdir("STAGED")
+    dcf = DockerContainerFactory(
+        simpleFileNameMethod=simpleTestContainerFileName,
+        containersCacheDir=cachepath.strpath,
+        stagedContainersDir=stagedpath.strpath,
+        tempDir=temppath.strpath,
+    )
+    containers = dcf.materializeContainers(tagList=[cont_tagged])
+    containers2 = dcf.materializeContainers(tagList=[cont_tagged], offline=True)
+    for container, container2 in zip(containers, containers2):
+        for attr in (
+            "origTaggedName",
+            "type",
+            "registries",
+            "taggedName",
+            "architecture",
+            "operatingSystem",
+            "fingerprint",
+            "source_type",
+        ):
+            assert getattr(container, attr) == getattr(
+                container2, attr
+            ), f"Expected and obtainer container '{attr}' do not match: {getattr(container, attr)} vs {getattr(container2, attr)}"