Separate tests between spec tests and infra tests AND change framework for test_template_test

.github/workflows/nightly-tests.yml

@@ -42,4 +42,4 @@ jobs:
         with:
           python-version: '3.13'
       - name: test-${{ matrix.fork }}
-        run: make test preset=mainnet fork=${{ matrix.fork }}
+        run: make test component=pyspec preset=mainnet fork=${{ matrix.fork }}

.github/workflows/release.yml

@@ -37,17 +37,23 @@ jobs:
             exit 1
           fi
 
+      # Ensure framework tests pass
+      - name: Run tests for test framework
+        run: |
+          cd consensus-specs
+          make test component=fw
+
       # Ensure minimal tests pass
       - name: Run tests for minimal
         run: |
           cd consensus-specs
-          make test preset=minimal
+          make test component=pyspec preset=minimal
 
       # Ensure mainnet tests pass
       - name: Run tests for mainnet
         run: |
           cd consensus-specs
-          make test preset=mainnet
+          make test component=pyspec preset=mainnet
 
       # Add support for large files
       - name: Install Git LFS

.github/workflows/run-tests.yml

@@ -53,8 +53,21 @@ jobs:
       - name: Check fork comments
         run: python3 scripts/check_fork_comments.py
 
+  framework:
+    runs-on: [self-hosted-ghr-custom, size-s-x64, profile-consensusSpecs]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Setup python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.13'
+          cache: 'pip'
+      - name: Run framework tests
+        run: make test component=fw
+
   tests:
-    needs: [lint, whitespace, comments]
+    needs: [lint, whitespace, comments, framework]
     runs-on: [self-hosted-ghr-custom, size-xl-x64, profile-consensusSpecs]
     strategy:
       matrix:
@@ -78,4 +91,4 @@ jobs:
           python-version: '3.13'
           cache: 'pip'
       - name: Run pyspec tests for ${{ matrix.fork }}
-        run: make test preset=minimal fork=${{ matrix.fork }}
+        run: make test component=pyspec preset=minimal fork=${{ matrix.fork }}

Makefile

@@ -66,10 +66,11 @@ help-verbose:
 	@echo "  by default using pytest with the minimal preset and fastest BLS library."
 	@echo ""
 	@echo "  Parameters:"
-	@echo "    k=<test>        Run specific test by name"
-	@echo "    fork=<fork>     Test specific fork (phase0, altair, bellatrix, capella, etc.)"
-	@echo "    preset=<preset> Use specific preset (mainnet or minimal; default: minimal)"
-	@echo "    bls=<type>      BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
+	@echo "    k=<test>          Run specific test by name"
+	@echo "    fork=<fork>       Test specific fork (phase0, altair, bellatrix, capella, etc.)"
+	@echo "    preset=<preset>   Use specific preset (mainnet or minimal; default: minimal)"
+	@echo "    bls=<type>        BLS library type (py_ecc, milagro, arkworks, fastest; default: fastest)"
+	@echo "    component=<value> Test component: (all, pyspec, fw; default: all)"
 	@echo ""
 	@echo "  Examples:"
 	@echo "    make test"
@@ -78,6 +79,7 @@ help-verbose:
 	@echo "    make test preset=mainnet"
 	@echo "    make test preset=mainnet fork=deneb k=test_verify_kzg_proof"
 	@echo "    make test bls=arkworks"
+	@echo "    make test component=fw"
 	@echo ""
 	@echo "$(BOLD)make coverage$(NORM)"
 	@echo ""
@@ -225,8 +227,10 @@ test: MAYBE_TEST := $(if $(k),-k=$(k))
 # Parallelism makes debugging difficult (print doesn't work).
 test: MAYBE_PARALLEL := $(if $(k),,-n auto)
 test: MAYBE_FORK := $(if $(fork),--fork=$(fork))
-test: PRESET := --preset=$(if $(preset),$(preset),minimal)
-test: BLS := --bls-type=$(if $(bls),$(bls),fastest)
+test: PRESET := $(if $(filter fw,$(component)),,--preset=$(if $(preset),$(preset),minimal))
+test: BLS := $(if $(filter fw,$(component)),,--bls-type=$(if $(bls),$(bls),fastest))
+test: MAYBE_ETH2SPEC := $(if $(filter fw,$(component)),,$(PYSPEC_DIR)/eth2spec)
+test: MAYBE_INFRA := $(if $(filter pyspec,$(component)),,$(CURDIR)/tests/infra)
 test: _pyspec
 	@mkdir -p $(TEST_REPORT_DIR)
 	@$(PYTHON_VENV) -m pytest \
@@ -237,8 +241,8 @@ test: _pyspec
 		$(PRESET) \
 		$(BLS) \
 		--junitxml=$(TEST_REPORT_DIR)/test_results.xml \
-		$(CURDIR)/tests/infra \
-		$(PYSPEC_DIR)/eth2spec
+		$(MAYBE_INFRA) \
+		$(MAYBE_ETH2SPEC)
 
 ###############################################################################
 # Coverage