feat: split nightly replays and nightly

Author: amkCha

.github/workflows/gradle-nightly-tests.yml

@@ -18,36 +18,31 @@ jobs:
     with:
       zkevm_fork: LONDON
       tests-with-ssh: ${{ inputs.tests-with-ssh || false }}
-      with-replays: true
 
   nightly-tests-paris:
     uses: ./.github/workflows/reusable-nightly-tests.yml
     secrets: inherit
     with:
       zkevm_fork: PARIS
       tests-with-ssh: ${{ inputs.tests-with-ssh || false }}
-      with-replays: false
 
   nightly-tests-shanghai:
     uses: ./.github/workflows/reusable-nightly-tests.yml
     secrets: inherit
     with:
       zkevm_fork: SHANGHAI
       tests-with-ssh: ${{ inputs.tests-with-ssh || false }}
-      with-replays: false
 
   nightly-tests-cancun:
     uses: ./.github/workflows/reusable-nightly-tests.yml
     secrets: inherit
     with:
       zkevm_fork: CANCUN
       tests-with-ssh: ${{ inputs.tests-with-ssh || false }}
-      with-replays: false
 
   nightly-tests-prague:
     uses: ./.github/workflows/reusable-nightly-tests.yml
     secrets: inherit
     with:
       zkevm_fork: PRAGUE
       tests-with-ssh: ${{ inputs.tests-with-ssh || false }}
-      with-replays: false

.github/workflows/gradle-replay-tests.yml

@@ -0,0 +1,51 @@
+name: Nightly Replay Tests
+permissions:
+  contents: read
+
+on:
+  schedule:
+    - cron: "0 22 * * 1-5"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # ==================================================================
+  # Replay Tests
+  # ==================================================================
+  nightly-replay-tests:
+    runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xxl
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          submodules: recursive
+
+      - name: Setup Test Environment
+        uses: ./.github/actions/setup-environment
+
+      - name: Run Nightly Replay tests
+        run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyReplayTests
+        env:
+          JAVA_OPTS: -Dorg.gradle.daemon=false
+          GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir
+          NIGHTLY_REPLAY_TESTS_PARALLELISM: 2
+
+      - name: Upload test report
+        if: ${{ always() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: nightly-replay-tests-report-${{ inputs.zkevm_fork }}
+          path: arithmetization/build/reports/tests/**/*
+
+      - name: Failure Notification
+        if: github.ref == 'refs/heads/arith-dev' && (failure() || cancelled())
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: webhook-trigger
+          payload: |
+            name: "NightlyReplay"
+            status: "${{ job.status }}"

.github/workflows/reusable-nightly-tests.yml

@@ -6,9 +6,6 @@ on:
       zkevm_fork:
         required: true
         type: string
-      with-replays:
-        required: true
-        type: boolean
       tests-with-ssh:
         required: true
         type: boolean
@@ -30,16 +27,6 @@ jobs:
 
       - name: Run Nightly tests
         run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyTests
-        if : ${{ inputs.with-replays == true }}
-        env:
-          JAVA_OPTS: -Dorg.gradle.daemon=false
-          GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir
-          NIGHTLY_TESTS_PARALLELISM: 2
-          ZKEVM_FORK: ${{ inputs.zkevm_fork }}
-
-      - name: Run Nightly No Replay tests
-        run: GOMAXPROCS=10 GOMEMLIMIT=20GiB ./gradlew nightlyNoReplayTests
-        if : ${{ inputs.with-replays == false }}
         env:
           JAVA_OPTS: -Dorg.gradle.daemon=false
           GOCORSET_FLAGS: -v --ansi-escapes=false --report --mir

arithmetization/src/test/java/net/consensys/linea/replaytests/HubShomeiReplayTests.java

@@ -25,7 +25,7 @@
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
 
-@Tag("nightly")
+@Tag("replay")
 @ExtendWith(UnitTestWatcher.class)
 public class HubShomeiReplayTests extends TracerTestBase {
 

arithmetization/src/test/java/net/consensys/linea/replaytests/Issue1123Tests.java

@@ -29,7 +29,6 @@
 @ExtendWith(UnitTestWatcher.class)
 public class Issue1123Tests extends TracerTestBase {
 
-  @Tag("nightly")
   @Test
   void issue_1123_mainnet_block_8043758(TestInfo testInfo) {
     replay(OLD_MAINNET_TESTCONFIG, "8043758.mainnet.json.gz", testInfo);

arithmetization/src/test/java/net/consensys/linea/replaytests/Issue1136Tests.java

@@ -42,7 +42,6 @@
  * <p><a href="https://github.com/Consensys/linea-tracer/issues/1153">Related GitHub issue</a>
  */
 @Tag("replay")
-@Tag("nightly")
 @ExtendWith(UnitTestWatcher.class)
 public class Issue1136Tests extends TracerTestBase {
 

arithmetization/src/test/java/net/consensys/linea/replaytests/Issue1216Tests.java

@@ -46,7 +46,6 @@ public class Issue1216Tests extends TracerTestBase {
    * blowing up with a NPE since the "after" version of the self destructor didn't exist yet in the
    * code. This was solved in issue #1216.
    */
-  @Tag("nightly")
   @Tag("replay")
   @Test
   void issue_1216_sepolia_block_2392659(TestInfo testInfo) {

arithmetization/src/test/java/net/consensys/linea/replaytests/Issue1264Tests.java

@@ -24,7 +24,6 @@
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
 
-@Tag("nightly")
 @Tag("replay")
 @ExtendWith(UnitTestWatcher.class)
 public class Issue1264Tests extends TracerTestBase {

arithmetization/src/test/java/net/consensys/linea/replaytests/ReplayTests.java

@@ -76,7 +76,6 @@ void multipleFailingCallToEcrecover(TestInfo testInfo) {
   }
 
   @Test
-  @Tag("nightly")
   void incident777zkGethMainnet(TestInfo testInfo) {
     replay(OLD_MAINNET_TESTCONFIG, "7461019-7461030.mainnet.json.gz", testInfo);
   }
@@ -163,7 +162,6 @@ void returnOogxForCodeDepositCost(TestInfo testInfo) {
   }
 
   @Test
-  @Tag("nightly")
   void modexpTriggeringNonAlignedFirstLimbSingleSourceMmuModexp(TestInfo testInfo) {
     replay(OLD_MAINNET_TESTCONFIG, "3108622-3108633.mainnet.json.gz", testInfo);
   }

gradle/tests.gradle

@@ -215,6 +215,10 @@ tasks.register("downloadShoemiBesuPlugin", Download) {
   overwrite false
 }
 
+/*
+  * Nightly replay tests (only nightly replay) and Nightly tests (no replay)
+ */
+
 tasks.register("fastReplayTests", Test) {
   if (System.getenv().containsKey("REPLAY_TESTS_PARALLELISM")) {
     systemProperty("junit.jupiter.execution.parallel.enabled", true)
@@ -278,43 +282,30 @@ tasks.register("jacocoFastReplayTestsReport", JacocoReport) {
 }
 
 tasks.register("nightlyReplayTests", Test) {
-  if (System.getenv().containsKey("REPLAY_TESTS_PARALLELISM")) {
+  if (System.getenv().containsKey("NIGHTLY_REPLAY_TESTS_PARALLELISM")) {
     systemProperty("junit.jupiter.execution.parallel.enabled", true)
     systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
     systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent")
     systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed")
     systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism",
-      System.getenv().getOrDefault("REPLAY_TESTS_PARALLELISM", "1").toInteger())
+      System.getenv().getOrDefault("NIGHTLY_REPLAY_TESTS_PARALLELISM", "1").toInteger())
   }
 
   useJUnitPlatform {
     includeTags("replay")
-    includeTags("nightly")
-  }
-}
-
-tasks.register("nightlyNoReplayTests", Test) {
-  systemProperty("junit.jupiter.execution.parallel.enabled", true)
-  systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
-  systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent")
-  systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed")
-  systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism",
-          System.getenv().getOrDefault("NIGHTLY_TESTS_PARALLELISM", "1").toInteger())
-
-  useJUnitPlatform {
-    includeTags("nightly")
-    excludeTags("replay")
+    excludeTags("weekly")
   }
-  finalizedBy(jacocoNightlyTestsReport)
 }
 
 tasks.register("nightlyTests", Test) {
+  if (System.getenv().containsKey("NIGHTLY_REPLAY_TESTS_PARALLELISM")) {
   systemProperty("junit.jupiter.execution.parallel.enabled", true)
   systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")
   systemProperty("junit.jupiter.execution.parallel.mode.classes.default", "concurrent")
   systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed")
   systemProperty("junit.jupiter.execution.parallel.config.fixed.parallelism",
           System.getenv().getOrDefault("NIGHTLY_TESTS_PARALLELISM", "1").toInteger())
+  }
 
   useJUnitPlatform {
     includeTags("nightly")
@@ -331,6 +322,11 @@ tasks.register("jacocoNightlyTestsReport", JacocoReport) {
   executionData(nightlyTests)
 }
 
+/*
+  * Weekly tests (include weekly replays and weekly tests)
+  * Prc Calls tests (only prc calls tests)
+ */
+
 tasks.register("weeklyTests", Test) {
   systemProperty("junit.jupiter.execution.parallel.enabled", true)
   systemProperty("junit.jupiter.execution.parallel.mode.default", "concurrent")