update tests

Author: bschimke95

.gitignore

@@ -30,3 +30,5 @@ squashfs-root
 
 subunit.html
 subunit.out
+
+.benchmark

tests/integration/requirements-test.txt

@@ -1,6 +1,7 @@
 coverage[toml]==7.2.5
 pytest==7.3.1
-pytest_tagging==1.5.3
+pytest_tagging==1.6.0
+pytest-benchmark==5.0.0
 PyYAML==6.0.1
 tenacity==8.2.3
 pylint==3.2.5

tests/integration/tests/conftest.py

@@ -14,7 +14,7 @@
 
 LOG = logging.getLogger(__name__)
 
-pytest_plugins = ("pytest_tagging", "pytest-benchmark")
+pytest_plugins = ("pytest_tagging","pytest_benchmark")
 
 
 def pytest_itemcollected(item):
@@ -24,11 +24,11 @@ def pytest_itemcollected(item):
     # Check for tags in the pytest.mark attributes
     marked_tags = [mark for mark in item.iter_markers(name="tags")]
     if not marked_tags or not any(
-        tag.args[0] in tags.TEST_LEVELS for tag in marked_tags
+        tag.args[0] in tags.TEST_TAGS for tag in marked_tags
     ):
         pytest.fail(
             f"The test {item.nodeid} does not have one of the test level tags."
-            f"Please add at least one test-level tag using @pytest.mark.tags ({tags.TEST_LEVELS})."
+            f"Please add at least one test tag using @pytest.mark.tags ({tags.TEST_TAGS})."
         )
 
 

tests/integration/tests/performance/create_benchmark_baselines.py

@@ -188,3 +188,6 @@
 
 # SONOBUOY_VERSION is the version of sonobuoy to use for CNCF conformance tests.
 SONOBUOY_VERSION = os.getenv("TEST_SONOBUOY_VERSION") or "v0.57.3"
+
+# PERF_DEFAULT_ROUNDS is the number of rounds to run for performance tests.
+PERF_DEFAULT_ROUNDS = int(os.getenv("TEST_PERF_DEFAULT_ROUNDS") or 5)

tests/integration/tests/test_util/config.py

@@ -8,8 +8,11 @@
 WEEKLY = "weekly"
 GPU = "gpu"
 CONFORMANCE = "conformance_tests"
+PERFORMANCE = "performance"
 
-TEST_LEVELS = [PULL_REQUEST, NIGHTLY, WEEKLY, CONFORMANCE]
+# Each test needs to be tagged with at least one of the following tags.
+# This is enforced by conftest.
+TEST_TAGS = [PULL_REQUEST, NIGHTLY, WEEKLY, CONFORMANCE, PERFORMANCE, GPU]
 
 # Those tags can be used for a convenient way to run multiple test levels.
 combine_tags("up_to_nightly", PULL_REQUEST, NIGHTLY)

tests/integration/tests/test_util/tags.py

@@ -47,12 +47,12 @@ commands_post =
 passenv =
     TEST_*
 
-[testenv:create_benchmark_baselines]
+[testenv:create_benchmark_baseline]
 description = Generate baselines for performance tests
 deps =
     -r {toxinidir}/requirements-test.txt
 commands =
-    python3 {toxinidir}/tests/performance/create_benchmark_baselines.py {posargs}
+    python3 {toxinidir}/tests/performance/create_baselines.py {posargs}
 passenv =
     TEST_*