diff --git a/.github/workflows/gradle-nightly-replay-tests.yml b/.github/workflows/gradle-nightly-replay-tests.yml new file mode 100644 index 0000000000..bf2143c347 --- /dev/null +++ b/.github/workflows/gradle-nightly-replay-tests.yml @@ -0,0 +1,51 @@ +name: Nightly Replay Tests +permissions: + contents: read + +on: + schedule: + - cron: "0 22 * * 1-5" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ================================================================== + # Replay Tests + # ================================================================== + nightly-replay-tests: + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + submodules: recursive + + - name: Setup Test Environment + uses: ./.github/actions/setup-environment + + - name: Run Nightly Replay tests + run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyReplayTests + env: + JAVA_OPTS: -Dorg.gradle.daemon=false + GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir + NIGHTLY_REPLAY_TESTS_PARALLELISM: 2 + + - name: Upload test report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nightly-replay-tests-report + path: arithmetization/build/reports/tests/**/* + + - name: Failure Notification + if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled()) + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: webhook-trigger + payload: | + name: "NightlyReplay" + status: "${{ job.status }}" diff --git a/.github/workflows/gradle-nightly-tests.yml b/.github/workflows/gradle-nightly-tests.yml index baa085d126..68ab0608e4 100644 --- a/.github/workflows/gradle-nightly-tests.yml +++ b/.github/workflows/gradle-nightly-tests.yml @@ -1,4 +1,6 @@ name: Nightly Tests +permissions: + contents: read on: schedule: @@ -10,52 +12,37 @@ concurrency: cancel-in-progress: true jobs: - nightly-tests: - runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl - steps: - - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - submodules: recursive - - - name: Setup Test Environment - uses: ./.github/actions/setup-environment - - - name: Run Nightly tests - run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyTests - env: - JAVA_OPTS: -Dorg.gradle.daemon=false - GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir - NIGHTLY_TESTS_PARALLELISM: 2 - ZKEVM_FORK: LONDON - - - name: Upload test report - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: nightly-tests-report-LONDON - path: arithmetization/build/reports/tests/**/* - - - name: Upload jacoco nightly tests coverage report - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: jacoco-nightly-tests-coverage-report-LONDON - path: arithmetization/build/reports/jacoco/jacocoNightlyTestsReport/**/* - - - name: Upload jacoco nightly tests exec file - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: jacoco-nightly-tests-exec-file-LONDON - path: arithmetization/build/jacoco/nightlyTests.exec - - - name: Failure Notification - if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled()) - uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 - with: - webhook: ${{ secrets.SLACK_WEBHOOK_URL }} - webhook-type: webhook-trigger - payload: | - name: "Nightly" - status: "${{ job.status }}" + nightly-tests-london: + uses: ./.github/workflows/reusable-nightly-tests.yml + secrets: inherit + with: + zkevm_fork: LONDON + tests-with-ssh: ${{ inputs.tests-with-ssh || false }} + + nightly-tests-paris: + uses: ./.github/workflows/reusable-nightly-tests.yml + secrets: inherit + with: + zkevm_fork: PARIS + tests-with-ssh: ${{ inputs.tests-with-ssh || false }} + + nightly-tests-shanghai: + uses: ./.github/workflows/reusable-nightly-tests.yml + secrets: inherit + with: + zkevm_fork: SHANGHAI + tests-with-ssh: ${{ inputs.tests-with-ssh || false }} + + nightly-tests-cancun: + uses: ./.github/workflows/reusable-nightly-tests.yml + secrets: inherit + with: + zkevm_fork: CANCUN + tests-with-ssh: ${{ inputs.tests-with-ssh || false }} + + nightly-tests-prague: + uses: ./.github/workflows/reusable-nightly-tests.yml + secrets: inherit + with: + zkevm_fork: PRAGUE + tests-with-ssh: ${{ inputs.tests-with-ssh || false }} diff --git a/.github/workflows/reusable-nightly-tests.yml b/.github/workflows/reusable-nightly-tests.yml new file mode 100644 index 0000000000..d57bf4b548 --- /dev/null +++ b/.github/workflows/reusable-nightly-tests.yml @@ -0,0 +1,65 @@ +name: "Reusable nightly tests workflow" + +on: + workflow_call: + inputs: + zkevm_fork: + required: true + type: string + tests-with-ssh: + required: true + type: boolean + +jobs: + # ================================================================== + # Nightly Tests + # ================================================================== + nightly-tests: + runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + submodules: recursive + + - name: Setup Test Environment + uses: ./.github/actions/setup-environment + + - name: Run Nightly tests + run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyTests + env: + JAVA_OPTS: -Dorg.gradle.daemon=false + GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir + NIGHTLY_TESTS_PARALLELISM: 2 + ZKEVM_FORK: ${{ inputs.zkevm_fork }} + + - name: Upload test report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: nightly-tests-report-${{ inputs.zkevm_fork }} + path: arithmetization/build/reports/tests/**/* + + - name: Upload jacoco nightly tests coverage report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: jacoco-nightly-tests-coverage-report-${{ inputs.zkevm_fork }} + path: arithmetization/build/reports/jacoco/jacocoNightlyTestsReport/**/* + + - name: Upload jacoco nightly tests exec file + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: jacoco-nightly-tests-exec-file-${{ inputs.zkevm_fork }} + path: arithmetization/build/jacoco/nightlyTests.exec + + - name: Failure Notification + if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled()) + uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 + with: + webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + webhook-type: webhook-trigger + payload: | + name: "Nightly" + status: "${{ job.status }}" diff --git a/Test Results - ExampleBesuTest_testPerFork.html b/Test Results - ExampleBesuTest_testPerFork.html deleted file mode 100644 index b9e4527942..0000000000 --- a/Test Results - ExampleBesuTest_testPerFork.html +++ /dev/null @@ -1,618 +0,0 @@ - - - - - Test Results — ExampleBesuTest.testPerFork - - - - - - - - - -
- -
- -
-
- - - diff --git a/Test Results - ForkTracingAndSwitchingBesuTest_testForkSwitchParisToCancun.html b/Test Results - ForkTracingAndSwitchingBesuTest_testForkSwitchParisToCancun.html deleted file mode 100644 index 21216f347e..0000000000 --- a/Test Results - ForkTracingAndSwitchingBesuTest_testForkSwitchParisToCancun.html +++ /dev/null @@ -1,1059 +0,0 @@ - - - - - Test Results — ForkTracingAndSwitchingBesuTest.testForkSwitchParisToCancun - - - - - - - - - -
- -
- -
-
- - - diff --git a/arithmetization/src/test/java/net/consensys/linea/replaytests/HubShomeiReplayTests.java b/arithmetization/src/test/java/net/consensys/linea/replaytests/HubShomeiReplayTests.java index a2759d2190..d1aeb2903e 100644 --- a/arithmetization/src/test/java/net/consensys/linea/replaytests/HubShomeiReplayTests.java +++ b/arithmetization/src/test/java/net/consensys/linea/replaytests/HubShomeiReplayTests.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; +@Tag("replay") @Tag("nightly") @ExtendWith(UnitTestWatcher.class) public class HubShomeiReplayTests extends TracerTestBase { diff --git a/gradle/tests.gradle b/gradle/tests.gradle index d0dc2ea1cc..3b63bcd4eb 100644 --- a/gradle/tests.gradle +++ b/gradle/tests.gradle @@ -215,6 +215,12 @@ tasks.register("downloadShoemiBesuPlugin", Download) { overwrite false } +/* + * Fast replay tests (only tagged with replay) + * Nightly replay tests (tagged with replay and nightly+replay) + * Nightly tests (no replay) + */ + tasks.register("fastReplayTests", Test) { if (System.getenv().containsKey("REPLAY_TESTS_PARALLELISM")) { systemProperty("junit.jupiter.execution.parallel.enabled", true) @@ -278,31 +284,34 @@ tasks.register("jacocoFastReplayTestsReport", JacocoReport) { } tasks.register("nightlyReplayTests", Test) { - if (System.getenv().containsKey("REPLAY_TESTS_PARALLELISM")) { + if (System.getenv().containsKey("NIGHTLY_REPLAY_TESTS_PARALLELISM")) { systemProperty("junit.jupiter.execution.parallel.enabled", true) systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", - System.getenv().getOrDefault("REPLAY_TESTS_PARALLELISM", "1").toInteger()) + System.getenv().getOrDefault("NIGHTLY_REPLAY_TESTS_PARALLELISM", "1").toInteger()) } useJUnitPlatform { includeTags("replay") - includeTags("nightly") + excludeTags("weekly") } } tasks.register("nightlyTests", Test) { + if (System.getenv().containsKey("NIGHTLY_TESTS_PARALLELISM")) { systemProperty("junit.jupiter.execution.parallel.enabled", true) systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent") systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent") systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed") systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism", System.getenv().getOrDefault("NIGHTLY_TESTS_PARALLELISM", "1").toInteger()) + } useJUnitPlatform { includeTags("nightly") + excludeTags("replay") } finalizedBy(jacocoNightlyTestsReport) } @@ -316,6 +325,11 @@ tasks.register("jacocoNightlyTestsReport", JacocoReport) { executionData(nightlyTests) } +/* + * Weekly tests (include weekly replays and weekly tests) + * Prc Calls tests (only prc calls tests) + */ + tasks.register("weeklyTests", Test) { systemProperty("junit.jupiter.execution.parallel.enabled", true) systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")