Validate config map and fix bad Fedora IoT test config

.github/workflows/gitlab-helper.yml

@@ -14,8 +14,12 @@ on:  # yamllint disable-line rule:truthy
 
 jobs:
   prepare:
-    name: "🔍 Check source preparation"
+    name: "🔍 Check source preparation and test configs"
     runs-on: ubuntu-latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     steps:
 
       - name: Set up Go 1.21
@@ -46,6 +50,11 @@ jobs:
             exit "0"
           fi
 
+      - name: Check that the config-map is valid
+        run: |
+          ./test/scripts/validate-config-map
+
+
   gitlab-ci-helper:
     name: "Gitlab CI trigger helper"
     runs-on: ubuntu-latest

.github/workflows/test-osbuild-composer-integration.yml

@@ -53,6 +53,10 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: registry.fedoraproject.org/fedora:latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     outputs:
       # Define job outputs
       # (see https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs#example-defining-outputs-for-a-job)

.github/workflows/tests.yml

@@ -23,19 +23,45 @@ jobs:
       fail-fast: false  # if one fails, keep the other(s) running
     name: "🛃 Unit tests (Fedora ${{ matrix.fedora_version }})"
     runs-on: ubuntu-latest
-
+    container: registry.fedoraproject.org/fedora:${{ matrix.fedora_version }}
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
+      - name: Install python3
+        # The Fedora 41 container doesn't have python3 installed by default
+        run: dnf -y install python3
+
+      - name: Set up repository for pinned osbuild commit
+        run: ./test/scripts/setup-osbuild-repo
+
+      - name: Install test dependencies
+        run: ./test/scripts/install-dependencies
+
+      - name: Mark the working directory as safe for git
+        run: git config --global --add safe.directory "$(pwd)"
+
       - name: Run unit tests
-        run: make BASE_CONTAINER_IMAGE_TAG=${{matrix.fedora_version}} gh-action-test
+        # skip the TestResolverLocalManifest test. It is tested separately
+        # (see below: requires root)
+        run: go test -race ./... -test.skip TestResolverLocalManifest
+
+      - name: Run depsolver tests with force-dnf to make sure it's not skipped
+        run: go test -race ./pkg/dnfjson/... -force-dnf
 
   container-resolver-tests:
     name: "🛃 Container resolver tests"
     runs-on: ubuntu-latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
 
     steps:
       - name: Set up Go 1.21
@@ -76,6 +102,9 @@ jobs:
       image: quay.io/centos/centos:${{ matrix.centos_stream.image_tag }}
     env:
       GOFLAGS: "-tags=exclude_graphdriver_btrfs"
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
 
     steps:
       - name: Install dnf plugins
@@ -114,6 +143,10 @@ jobs:
   lint:
     name: "⌨ Lint"
     runs-on: ubuntu-latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     steps:
       - name: Set up Go 1.21
         uses: actions/setup-go@v5
@@ -146,6 +179,10 @@ jobs:
   shellcheck:
     name: "🐚 Shellcheck"
     runs-on: ubuntu-latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     steps:
       - uses: actions/checkout@v4
         with:
@@ -160,27 +197,35 @@ jobs:
           SHELLCHECK_OPTS: -e SC1091 -e SC2002 -e SC2317
 
   python-test:
-    name: "🐍 pytest (imgtestlib)"
+    name: "🐍 pytest (imgtestlib and test scripts)"
     runs-on: ubuntu-latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     container:
       image: registry.fedoraproject.org/fedora:latest
     steps:
 
       - name: Install build and test dependencies
-        run: dnf -y install python3-pytest podman skopeo
+        run: dnf -y install python3-pytest podman skopeo go btrfs-progs-devel device-mapper-devel gpgme-devel
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Testing imgtestlib
+      - name: Testing imgtestlib and test scripts
         run: |
           python3 -m pytest -v
 
   python-lint:
     name: "🐍 Lint (test scripts)"
     runs-on: ubuntu-latest
+    env:
+      # workaround for expired cert at source of indirect dependency
+      # (go.opencensus.io/trace)
+      GOPROXY: "https://proxy.golang.org|direct"
     container:
       image: registry.fedoraproject.org/fedora:latest
     steps:

test/config-map.json

@@ -168,7 +168,7 @@
   },
   "./configs/iot-ostree-pull-user.json": {
     "image-types": [
-      "iot-ami"
+      "iot-qcow2-image"
     ]
   },
   "./configs/kernel-debug.json": {

test/configs/edge-ostree-pull-user.json

@@ -8,7 +8,15 @@
             "wheel"
           ],
           "key": "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNebAh6SjpAn8wB53K4695cGnHGuCtl4RdaX3futZgJUultHyzeYHnzMO7d4++qnRL+Rworew62LKP560uvtncc= github.com/osbuild/images",
-          "name": "osbuild"
+          "name": "osbuild",
+          "uid": 1001,
+          "gid": 1002
+        }
+      ],
+      "group": [
+        {
+          "name": "osbuild",
+          "gid": 1002
         }
       ]
     }

test/scripts/install-dependencies

@@ -7,6 +7,7 @@ dnf -y install \
     gcc \
     go \
     gpgme-devel \
+    krb5-devel \
     osbuild \
     osbuild-depsolve-dnf \
     osbuild-luks2 \

test/scripts/test_validate_config_map.py

@@ -0,0 +1,190 @@
+import importlib
+from types import ModuleType
+
+import pytest
+
+
+def import_validator() -> ModuleType:
+    name = "validate_config_map"
+    path = "test/scripts/validate-config-map"
+    loader = importlib.machinery.SourceFileLoader(name, path)
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    if spec is None:
+        raise ImportError(f"cannot import {name} from {path}, got None as the spec")
+    mod = importlib.util.module_from_spec(spec)
+    loader.exec_module(mod)
+    return mod
+
+
+def write_configs(files, root):
+    for config_file in files:
+        config_path = root / config_file
+        config_path.parent.mkdir(exist_ok=True)
+        config_path.touch()
+
+
+@pytest.mark.parametrize("config_map,files,missing_files,invalid_cfgs", (
+    # valid
+    (
+        {
+            "everything.json": {
+                "distros": [
+                    "fedora*",
+                ],
+            },
+            "empty.json": {
+                "image-types": ["qcow2"],
+            },
+        },
+        ["everything.json", "empty.json"],
+        [],
+        [],
+    ),
+    (
+        {
+            "configs/cfg-1.json": {},
+            "configs/cfg-2.json": {
+                "distros": ["centos*"],
+                "arches": ["s390x"],
+                "image-types": ["qcow2"],
+            },
+        },
+        ["configs/cfg-1.json", "configs/cfg-2.json"],
+        [],
+        [],
+    ),
+    (
+        {
+            "configs/cfg-3.json": {
+                "distros": ["fedora*"],
+            },
+            "configs/cfg-4.json": {
+                "image-types": ["qcow2"],
+            },
+        },
+        ["configs/cfg-3.json", "configs/cfg-4.json"],
+        [],
+        [],
+    ),
+
+    # missing files
+    (
+        {
+            "everything.json": {
+                "distros": [
+                    "fedora*",
+                ],
+            },
+            "empty.json": {
+                "image-types": ["qcow2"],
+            },
+        },
+        ["everything.json"],
+        ["empty.json"],
+        [],
+    ),
+    (
+        {
+            "configs/cfg-1.json": {},
+            "configs/cfg-2.json": {
+                "distros": ["centos*"],
+                "arches": ["s390x"],
+                "image-types": ["qcow2"],
+            },
+        },
+        [],
+        ["configs/cfg-1.json", "configs/cfg-2.json"],
+        [],
+    ),
+    (
+        {
+            "configs/cfg-3.json": {
+                "distros": ["fedora*"],
+            },
+            "configs/cfg-4.json": {
+                "image-types": ["qcow2"],
+            },
+        },
+        ["configs/cfg-4.json"],
+        ["configs/cfg-3.json"],
+        [],
+    ),
+
+    # bad config
+    (
+        {
+            "everything.json": {
+                "distros": [
+                    "fedora*",
+                ],
+            },
+            "empty.json": {
+                "image-types": ["not-qcow2"],
+            },
+        },
+        ["everything.json", "empty.json"],
+        [],
+        [
+            (
+                "empty.json",
+                {
+                    "image-types": ["not-qcow2"],
+                },
+            )
+        ],
+    ),
+    (
+        {
+            "configs/cfg-1.json": {},
+            "configs/cfg-2.json": {
+                "distros": ["centos*"],
+                "arches": ["noarch"],
+                "image-types": ["qcow2"],
+            },
+        },
+        ["configs/cfg-1.json", "configs/cfg-2.json"],
+        [],
+        [
+            (
+                "configs/cfg-2.json",
+                {
+                    "distros": ["centos*"],
+                    "arches": ["noarch"],
+                    "image-types": ["qcow2"],
+                },
+            )
+        ],
+    ),
+    (
+        {
+            "configs/cfg-3.json": {
+                "distros": ["archlinux"],
+            },
+            "configs/cfg-4.json": {
+                "distros": ["ubuntu*"],
+            },
+        },
+        ["configs/cfg-3.json", "configs/cfg-4.json"],
+        [],
+        [
+            (
+                "configs/cfg-3.json",
+                {
+                    "distros": ["archlinux"],
+                },
+            ),
+            (
+                "configs/cfg-4.json",
+                {
+                    "distros": ["ubuntu*"],
+                },
+            ),
+        ],
+    ),
+))
+def test_valid_config_map(config_map, files, missing_files, invalid_cfgs, tmp_path):
+    validator = import_validator()
+    write_configs(files, tmp_path)
+
+    assert validator.validate_config_file_paths(config_map, tmp_path) == [tmp_path / mf for mf in missing_files]
+    assert validator.validate_build_config(config_map) == invalid_cfgs