feat: performance tests

Author: bschimke95

.github/workflows/create-perf-benchmarks.yaml

@@ -0,0 +1,57 @@
+name: Create Performance Benchmarks
+description: |
+  This workflow runs performance benchmarks on the specified runner tags and tests
+  and creates a baseline for future comparisons. It uses pytest-benchmark to run the tests
+  and save the results.
+  It can also create a pull request with the results if specified.
+
+on:
+  workflow_dispatch:
+    inputs:
+      create_pr:
+        description: "Create PR with results (true/false)"
+        required: false
+        default: "false"
+  pull_request:
+    path: |
+      .github/workflows/create-perf-benchmarks.yaml
+
+jobs:
+  benchmark:
+    name: Run Benchmarks for ${{ matrix.os }}-${{ matrix.arch }}
+    strategy:
+      matrix:
+        os: ["ubuntu:22.04", "ubuntu:24.04"]
+        arch: ["amd64", "arm64"]
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run benchmarks
+        run: |
+          tox -e create_benchmark_baseline -- --baseline-directory=tests/integration/tests/performance/benchmarks --baseline-name=${ matrix.os }-${{ matrix.arch }}
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: benchmark-results
+          path: tests/integration/
+
+      - name: Commit and create PR
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: "Add benchmark baseline results"
+          title: "Benchmark Baselines"
+          body: "Automated benchmark results from manual run"
+          branch: benchmark/baseline-results
+          labels: |
+            benchmark
+            automation
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/nightly-test.yaml

@@ -13,20 +13,33 @@ permissions:
   contents: read
 
 jobs:
-  test-integration:
-    name: Integration
+  test-performance:
+    name: Run performance tests for ${{ matrix.os }}-${{ matrix.arch }}
     strategy:
       matrix:
         os: ["ubuntu:22.04", "ubuntu:24.04"]
         arch: ["amd64", "arm64"]
-        channel: ["latest/edge"]
-      fail-fast: false # TODO: remove once we no longer have flaky tests.
-    uses: ./.github/workflows/e2e-tests.yaml
-    with:
-      arch: ${{ matrix.arch }}
-      os: ${{ matrix.os }}
-      channel: ${{ matrix.channel }}
-      test-tags: 'up_to_nightly conformance_tests'
+    runs-on: ${{ matrix.arch == 'arm64' && 'self-hosted-linux-arm64-noble-large' || 'self-hosted-linux-amd64-noble-large' }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Run benchmarks
+        env:
+          TEST_SUBSTRATE: lxd
+          TEST_LXD_IMAGE: ${{ matrix.os }}
+        run: |
+
+          tox -e create_benchmark_baseline -- --baseline-directory=benchmark-results --baseline-name=${ matrix.os }-${{ matrix.arch }}
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: benchmark-results/
 
   Trivy:
     permissions:

tests/integration/tests/conftest.py

@@ -14,7 +14,7 @@
 
 LOG = logging.getLogger(__name__)
 
-pytest_plugins = ("pytest_tagging",)
+pytest_plugins = ("pytest_tagging", "pytest-benchmark")
 
 
 def pytest_itemcollected(item):

tests/integration/tests/performance/create_benchmark_baselines.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import subprocess
+
+
+def _prepare_baseline_directory(baseline_directory: str, baseline_name: str):
+    """
+    Prepare the baseline directory by removing existing baselines with the
+    specified baseline name and creating the directory if it doesn't exist.
+    """
+    if os.path.exists(baseline_directory):
+        for root, _, files in os.walk(baseline_directory):
+            for file in files:
+                file_path = os.path.join(root, file)
+                if baseline_name in file:
+                    os.unlink(file_path)
+    os.makedirs(baseline_directory, exist_ok=True)
+
+    return baseline_directory
+
+
+def run_benchmarks(baseline_directory: str, baseline_name: str):
+    print(
+        f"Running benchmark tests and store results as {baseline_directory}/{baseline_name}"
+    )
+
+    subprocess.run(
+        [
+            "tox",
+            "-e",
+            "integration",
+            "--",
+            "--tags",
+            "performance",
+            f"--benchmark-save={baseline_name}",
+            f"--benchmark-storage={baseline_directory}",
+        ],
+        check=True,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Create performance benchmarks using pytest-benchmark."
+    )
+    parser.add_argument(
+        "--baseline-name", help="Name for saved benchmark results", required=True
+    )
+    parser.add_argument(
+        "--baseline-directory",
+        help="Directory for saved benchmark results",
+        required=True,
+    )
+    args = parser.parse_args()
+    _prepare_baseline_directory(args.baseline_directory, args.baseline_name)
+    run_benchmarks(args.baseline_directory, args.baseline_name)
+    print(f"All benchmarks saved in {args.baseline_directory}/{args.baseline_name}.")
+
+
+if __name__ == "__main__":
+    main()

tests/integration/tests/performance/test_perf_clustering.py

@@ -0,0 +1,21 @@
+import pytest
+from typing import List
+from test_util import config, harness, tags, util
+
+@pytest.mark.node_count(1)
+@pytest.mark.no_setup()
+@pytest.mark.tags(tags.PERFORMANCE)
+def test_perf_clustering_bootstrap_cli(instances: List[harness.Instance], tmp_path: str, benchmark):
+    node = instances[0]
+
+    def setup():
+        # TODO(ben): benchmark `teardown` function is implemented but not yet released
+        # in 5.1.0. Once released, we can move this teardown logic in a separate function.
+        # See https://github.com/ionelmc/pytest-benchmark/issues/270
+        node.exec(["snap", "remove", "k8s", "--purge"])
+        util.setup_k8s_snap(node, tmp_path)
+
+    def run():
+        node.exec(["k8s", "bootstrap"])
+
+    benchmark.pedantic(run, setup=setup, rounds=config.PERF_DEFAULT_ROUNDS)

tests/integration/tests/performance/test_perf_status.py

@@ -0,0 +1,20 @@
+import pytest
+from typing import List
+from test_util import config, harness, tags, util
+
+@pytest.mark.node_count(1)
+@pytest.mark.tags(tags.PULL_REQUEST, tags.PERFORMANCE)
+def test_perf_status_single_node_cli(instances: List[harness.Instance], tmp_path: str, benchmark):
+    node = instances[0]
+
+    def run():
+        node.exec(["k8s", "status"])
+
+    benchmark.pedantic(run, rounds=20)
+
+@pytest.mark.tags(tags.PULL_REQUEST, tags.PERFORMANCE)
+def test_perf_short(benchmark):
+    def run():
+        print("short")
+
+    benchmark.pedantic(run, rounds=20)

tests/integration/tox.ini

@@ -44,7 +44,15 @@ commands =
         {posargs}
 commands_post =
     subunit2html "{toxinidir}/subunit.out" "{toxinidir}/subunit.html"
+passenv =
+    TEST_*
 
+[testenv:create_benchmark_baselines]
+description = Generate baselines for performance tests
+deps =
+    -r {toxinidir}/requirements-test.txt
+commands =
+    python3 {toxinidir}/tests/performance/create_benchmark_baselines.py {posargs}
 passenv =
     TEST_*