Add Allure test reporting for tracking test progress

.github/workflows/python-tests.yaml

@@ -93,11 +93,50 @@ jobs:
               "https://iad.mirror.rackspace.com/fedora/releases/43/Cloud/${arch}/images/Fedora-Cloud-Base-Generic-43-1.6.${arch}.qcow2"
           done
 
-      - name: Run pytest
+      - name: Run pytest with Allure reporting
         working-directory: python
-        run: |
-            make test
+        run: make test-allure
+
+      - name: Upload Allure results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure-results-${{ matrix.runs-on }}-py${{ matrix.python-version }}
+          path: python/allure-results/
+          retention-days: 30
+          if-no-files-found: ignore
+
+  allure-report:
+    needs: [changes, pytest-matrix]
+    if: always() && needs.pytest-matrix.result != 'skipped'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Download all Allure results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: allure-results-*
+          path: allure-results
+          merge-multiple: true
 
+      - name: Generate Allure report
+        run: |
+          if [ -d "allure-results" ] && [ "$(ls -A allure-results 2>/dev/null)" ]; then
+            npm install -g allure-commandline@2.32.0
+            allure generate allure-results -o allure-report --clean
+          else
+            echo "No Allure results found, skipping report generation"
+          fi
+
+      - name: Upload Allure report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: allure-report
+          path: allure-report/
+          retention-days: 30
+          if-no-files-found: ignore
 
   # https://github.com/orgs/community/discussions/26822
   pytest:

python/.gitignore

@@ -162,6 +162,10 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
+# Allure test reporting
+allure-results/
+allure-report/
+
 # Ruff cache
 .ruff_cache/
 mitmproxy-ca-cert.pem

python/Makefile

@@ -24,6 +24,9 @@ help:
 	@echo "  pkg-test-all       - Run tests for all packages"
 	@echo "  pkg-test-<pkg>     - Run tests for a specific package"
 	@echo "  docs-test          - Run documentation tests"
+	@echo "  test-allure        - Run all tests with Allure reporting"
+	@echo "  allure-serve       - Serve the Allure report locally (requires: npm i -g allure-commandline)"
+	@echo "  allure-report      - Generate a static Allure report (requires: npm i -g allure-commandline)"
 	@echo ""
 	@echo "Linting and type checking:"
 	@echo "  ty                 - Run ty type checking on all packages"
@@ -62,14 +65,21 @@ docs-test:
 docs-linkcheck:
 	uv run --isolated --all-packages --group docs $(MAKE) -C docs linkcheck
 
+ALLURE_RESULTS_DIR ?= $(CURDIR)/allure-results
+
 pkg-test-%: packages/%
 	uv run --isolated --directory $< pytest || [ $$? -eq 5 ]
 
+pkg-test-allure-%: packages/%
+	uv run --isolated --directory $< --with allure-pytest pytest --alluredir=$(ALLURE_RESULTS_DIR) || [ $$? -eq 5 ]
+
 pkg-ty-%: packages/%
 	uv run --isolated --directory $< ty check .
 
 pkg-test-all: $(addprefix pkg-test-,$(PKG_TARGETS))
 
+pkg-test-allure-all: $(addprefix pkg-test-allure-,$(PKG_TARGETS))
+
 pkg-ty-all: $(addprefix pkg-ty-,$(PKG_TARGETS))
 
 build:
@@ -102,6 +112,8 @@ clean-test:
 	-rm -f .coverage
 	-rm -f coverage.xml
 	-rm -rf htmlcov
+	-rm -rf allure-results
+	-rm -rf allure-report
 
 clean-docs:
 	uv run --isolated --all-packages --group docs $(MAKE) -C docs clean
@@ -110,6 +122,19 @@ clean: clean-docs clean-venv clean-build clean-test
 
 test: pkg-test-all docs-test
 
+test-allure: clean-allure pkg-test-allure-all docs-test
+
+clean-allure:
+	-rm -rf $(ALLURE_RESULTS_DIR)
+
+allure-serve:
+	@command -v allure >/dev/null 2>&1 || { echo "Allure CLI required: npm install -g allure-commandline"; exit 1; }
+	allure serve $(ALLURE_RESULTS_DIR)
+
+allure-report:
+	@command -v allure >/dev/null 2>&1 || { echo "Allure CLI required: npm install -g allure-commandline"; exit 1; }
+	allure generate $(ALLURE_RESULTS_DIR) -o allure-report --clean
+
 ty: pkg-ty-all
 
 lint:
@@ -119,9 +144,9 @@ lint-fix:
 	uv run ruff check --fix
 
 .PHONY: default help docs docs-all docs-serve docs-serve-all docs-clean docs-test \
-	docs-linkcheck pkg-test-all pkg-ty-all build generate sync \
-	clean-venv clean-build clean-test clean-all test-all ty-all docs \
-	lint lint-fix \
+	docs-linkcheck pkg-test-all pkg-test-allure-all pkg-ty-all build generate sync \
+	clean-venv clean-build clean-test clean-allure clean-all test-all test-allure ty-all docs \
+	lint lint-fix allure-serve allure-report \
 	pkg-ty-jumpstarter \
 	pkg-ty-jumpstarter-cli-admin \
 	pkg-ty-jumpstarter-kubernetes \

python/conftest.py

@@ -1,5 +1,8 @@
 import os
+import platform
+import sys
 from contextlib import contextmanager
+from pathlib import Path
 
 import pytest
 
@@ -32,3 +35,60 @@ def tmp_config_path(tmp_path, monkeypatch):
     def console_size(monkeypatch):
         monkeypatch.setenv("COLUMNS", "1024")
         monkeypatch.setenv("LINES", "1024")
+
+
+# ---------------------------------------------------------------------------
+# Allure integration: auto-label tests by package and component
+# ---------------------------------------------------------------------------
+_PACKAGE_SUITES = {
+    "jumpstarter": "Core",
+    "jumpstarter-protocol": "Core",
+    "jumpstarter-testing": "Testing",
+    "jumpstarter-kubernetes": "Kubernetes",
+    "jumpstarter-imagehash": "Utilities",
+}
+
+try:
+    import allure
+
+    def _extract_package_name(item):
+        path_str = str(item.path)
+        if "/packages/" in path_str:
+            return path_str.split("/packages/")[1].split("/")[0]
+        return None
+
+    def _classify_package(pkg_name):
+        if pkg_name in _PACKAGE_SUITES:
+            return _PACKAGE_SUITES[pkg_name]
+        if pkg_name.startswith("jumpstarter-driver-"):
+            return "Drivers"
+        if pkg_name.startswith("jumpstarter-cli"):
+            return "CLI"
+        return "Other"
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_collection_modifyitems(items):
+        for item in items:
+            pkg = _extract_package_name(item)
+            if pkg:
+                item.add_marker(allure.parent_suite(_classify_package(pkg)))
+                item.add_marker(allure.suite(pkg))
+
+    def pytest_sessionstart(session):
+        alluredir = session.config.getoption("alluredir", default=None)
+        if alluredir:
+            results_dir = Path(alluredir)
+            results_dir.mkdir(parents=True, exist_ok=True)
+            env_file = results_dir / "environment.properties"
+            props = {
+                "Python": sys.version.split()[0],
+                "Platform": platform.platform(),
+                "OS": platform.system(),
+                "Architecture": platform.machine(),
+            }
+            with open(env_file, "w", encoding="utf-8") as f:
+                for key, value in props.items():
+                    f.write(f"{key}={value}\n")
+
+except ImportError:
+    pass

python/packages/jumpstarter-driver-network/jumpstarter_driver_network/driver_test.py

@@ -2,10 +2,22 @@
 import socket
 import subprocess
 import sys
+import types
 from shutil import which
 from unittest.mock import AsyncMock, patch
 
 import pytest
+
+try:
+    import allure
+except ImportError:
+    _noop = lambda *a, **kw: lambda f: f
+    allure = types.SimpleNamespace(
+        feature=_noop,
+        story=_noop,
+        severity=_noop,
+        severity_level=types.SimpleNamespace(CRITICAL="critical", MINOR="minor"),
+    )
 from anyio.from_thread import start_blocking_portal
 from click.testing import CliRunner
 
@@ -24,6 +36,8 @@
                 pass
 
 
+@allure.feature("TCP")
+@allure.story("Port forwarding")
 def test_tcp_network_portforward(tcp_echo_server):
     with serve(TcpNetwork(host=tcp_echo_server[0], port=tcp_echo_server[1])) as client:
         with TcpPortforwardAdapter(client=client) as addr:
@@ -33,6 +47,8 @@
             assert stream.recv(5) == b"hello"
 
 
+@allure.feature("Unix socket")
+@allure.story("Port forwarding")
 def test_unix_network_portforward():
     with start_blocking_portal() as portal:
         with portal.wrap_async_context_manager(TemporaryUnixListener(echo_handler)) as inner:
@@ -72,6 +88,9 @@
                     assert stream.receive() == b"hello"
 
 
+@allure.feature("TCP")
+@allure.story("Performance")
+@allure.severity(allure.severity_level.MINOR)
 @pytest.mark.skipif(which("iperf3") is None, reason="iperf3 not available")
 def test_tcp_network_performance():
     with serve(
@@ -169,6 +188,7 @@
         assert oldvar == os.getenv("DBUS_SESSION_BUS_ADDRESS")
 
 
+@allure.feature("WebSocket")
 @pytest.mark.asyncio
 async def test_websocket_network_connect():
     ws = AsyncMock()

python/packages/jumpstarter-testing/jumpstarter_testing/pytest_test.py

@@ -1,11 +1,25 @@
+import types
+
 from jumpstarter_driver_power.driver import MockPower
 from pytest import Pytester
 
 from jumpstarter.common import ExporterStatus
 from jumpstarter.config.env import JMP_DRIVERS_ALLOW, JUMPSTARTER_HOST
 from jumpstarter.exporter import Session
 
+try:
+    import allure
+except ImportError:
+    _noop = lambda *a, **kw: lambda f: f
+    allure = types.SimpleNamespace(
+        feature=_noop,
+        severity=_noop,
+        severity_level=types.SimpleNamespace(CRITICAL="critical"),
+    )
+
 
+@allure.feature("pytest integration")
+@allure.severity(allure.severity_level.CRITICAL)
 def test_env(pytester: Pytester, monkeypatch):
     pytester.makepyfile(
         """

python/pyproject.toml

@@ -74,6 +74,7 @@ dev = [
     "pre-commit>=3.8.0",
     "esbonio>=0.16.5",
     "ty>=0.0.1a8",
+    "allure-pytest>=2.13.0",
 ]
 
 [tool.ruff]