Merge pull request #2342 from bugsnag/PLAT-13707/rn-test-matrix

Update React Native test matrix

.buildkite/full/react-native-android-pipeline.full.yml

@@ -26,42 +26,6 @@ steps:
       #
       # Test fixtures
       #
-      - label: ":android: Build RN 0.66 apk"
-        key: "rn-0-66-apk"
-        depends_on:
-          - "android-builder-image-java-11-node-16"
-        timeout_in_minutes: 20
-        env:
-          REACT_NATIVE_VERSION: "rn0.66"
-        plugins:
-          - docker-compose#v4.12.0:
-              pull: react-native-android-builder-java-11-node-16
-              run: react-native-android-builder-java-11-node-16
-        artifact_paths:
-          - build/rn0.66.apk
-        retry:
-          automatic:
-            - exit_status: "*"
-              limit: 1
-
-      - label: ":android: Build RN 0.67 apk"
-        key: "rn-0-67-apk"
-        depends_on:
-          - "android-builder-image-java-11-node-16"
-        timeout_in_minutes: 20
-        env:
-          REACT_NATIVE_VERSION: "rn0.67"
-        plugins:
-          - docker-compose#v4.12.0:
-              pull: react-native-android-builder-java-11-node-16
-              run: react-native-android-builder-java-11-node-16
-        artifact_paths:
-          - build/rn0.67.apk
-        retry:
-          automatic:
-            - exit_status: "*"
-              limit: 1
-
       - label: ":android: Build RN 0.68 Hermes apk"
         key: "rn-0-68-hermes-apk"
         depends_on:
@@ -115,9 +79,7 @@ steps:
           - "bundle install"
           - "node scripts/generate-react-native-fixture.js"
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"
@@ -147,7 +109,6 @@ steps:
             - exit_status: "*"
               limit: 1
         matrix:
-          - "0.71"
           - "0.72"
           - "0.73"
           - "0.74"
@@ -207,51 +168,19 @@ steps:
       #
       # End-to-end tests
       #
-      - label: ":bitbar: :android: RN 0.66 Android end-to-end tests"
-        depends_on: "rn-0-66-apk"
-        timeout_in_minutes: 30
-        plugins:
-          artifacts#v1.9.0:
-            download: "build/rn0.66.apk"
-            upload: ./test/react-native/maze_output/**/*
-          docker-compose#v4.7.0:
-            pull: react-native-maze-runner
-            run: react-native-maze-runner
-            service-ports: true
-            command:
-              - --app=build/rn0.66.apk
-              - --app-activity=com.reactnative.MainActivity
-              - --app-package=com.reactnative
-              - --farm=bb
-              - --device=ANDROID_10|ANDROID_11|ANDROID_12
-              - --appium-version=1.22
-              - --fail-fast
-              - --no-tunnel
-              - --aws-public-ip
-          test-collector#v1.10.2:
-            files: "reports/TEST-*.xml"
-            format: "junit"
-            branch: "^main|next$$"
-            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
-        env:
-          SKIP_NAVIGATION_SCENARIOS: "true"
-        concurrency: 25
-        concurrency_group: 'bitbar'
-        concurrency_method: eager
-
-      - label: ":bitbar: :android: RN 0.67 Android end-to-end tests"
-        depends_on: "rn-0-67-apk"
+      - label: ":bitbar: :android: RN 0.68 (Hermes) Android end-to-end tests"
+        depends_on: "rn-0-68-hermes-apk"
         timeout_in_minutes: 30
         plugins:
           artifacts#v1.9.0:
-            download: "build/rn0.67.apk"
+            download: "build/rn0.68-hermes.apk"
             upload: ./test/react-native/maze_output/**/*
           docker-compose#v4.7.0:
             pull: react-native-maze-runner
             run: react-native-maze-runner
             service-ports: true
             command:
-              - --app=build/rn0.67.apk
+              - --app=build/rn0.68-hermes.apk
               - --app-activity=com.reactnative.MainActivity
               - --app-package=com.reactnative
               - --farm=bb
@@ -305,39 +234,6 @@ steps:
         concurrency_group: 'bitbar'
         concurrency_method: eager
 
-      - label: ":bitbar: :android: RN 0.68 (Hermes) Android end-to-end tests"
-        depends_on: "rn-0-68-hermes-apk"
-        timeout_in_minutes: 30
-        plugins:
-          artifacts#v1.9.0:
-            download: "build/rn0.68-hermes.apk"
-            upload: ./test/react-native/maze_output/**/*
-          docker-compose#v4.7.0:
-            pull: react-native-maze-runner
-            run: react-native-maze-runner
-            service-ports: true
-            command:
-              - --app=build/rn0.68-hermes.apk
-              - --app-activity=com.reactnative.MainActivity
-              - --app-package=com.reactnative
-              - --farm=bb
-              - --device=ANDROID_10|ANDROID_11|ANDROID_12
-              - --appium-version=1.22
-              - --fail-fast
-              - --no-tunnel
-              - --aws-public-ip
-          test-collector#v1.10.2:
-            files: "reports/TEST-*.xml"
-            format: "junit"
-            branch: "^main|next$$"
-            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
-        env:
-          SKIP_NAVIGATION_SCENARIOS: "true"
-          HERMES: "true"
-        concurrency: 25
-        concurrency_group: 'bitbar'
-        concurrency_method: eager
-
       - label: ":bitbar: :android: RN {{matrix}} Android 12 (Old Arch) end-to-end tests"
         depends_on: "build-react-native-android-fixture-old-arch-full"
         timeout_in_minutes: 30
@@ -372,9 +268,7 @@ steps:
         concurrency_group: "bitbar"
         concurrency_method: eager
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"
@@ -416,9 +310,7 @@ steps:
         concurrency_group: "bitbar"
         concurrency_method: eager
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"

.buildkite/full/react-native-ios-pipeline.full.yml

@@ -9,40 +9,6 @@ steps:
       #
       # Test fixtures
       #
-      - label: ":ios: Build RN 0.66 ipa"
-        key: "rn-0-66-ipa"
-        timeout_in_minutes: 20
-        agents:
-          queue: "macos-12-arm"
-        env:
-          REACT_NATIVE_VERSION: rn0.66
-          LANG: "en_US.UTF-8"
-          DEVELOPER_DIR: "/Applications/Xcode14.app"
-        artifact_paths: build/rn0.66.ipa
-        commands:
-          - npm run test:build-react-native-ios
-        retry:
-          automatic:
-            - exit_status: "*"
-              limit: 1
-
-      - label: ":ios: Build RN 0.67 ipa"
-        key: "rn-0-67-ipa"
-        timeout_in_minutes: 20
-        agents:
-          queue: "macos-12-arm"
-        env:
-          REACT_NATIVE_VERSION: rn0.67
-          LANG: "en_US.UTF-8"
-          DEVELOPER_DIR: "/Applications/Xcode14.app"
-        artifact_paths: build/rn0.67.ipa
-        commands:
-          - npm run test:build-react-native-ios
-        retry:
-          automatic:
-            - exit_status: "*"
-              limit: 1
-
       - label: ":ios: Build RN 0.68 Hermes ipa"
         key: "rn-0-68-hermes-ipa"
         timeout_in_minutes: 20
@@ -94,9 +60,7 @@ steps:
           - "bundle install"
           - "node scripts/generate-react-native-fixture.js"
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"
@@ -122,9 +86,7 @@ steps:
           - "bundle install"
           - "node scripts/generate-react-native-fixture.js"
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"
@@ -188,50 +150,20 @@ steps:
       #
       # End-to-end tests
       #
-      - label: ":bitbar: :ios: RN 0.66 iOS end-to-end tests"
-        depends_on: "rn-0-66-ipa"
-        timeout_in_minutes: 40
-        plugins:
-          artifacts#v1.5.0:
-            download: "build/rn0.66.ipa"
-            upload: ./test/react-native/maze_output/**/*
-          docker-compose#v4.12.0:
-            pull: react-native-maze-runner
-            run: react-native-maze-runner
-            service-ports: true
-            use-aliases: true
-            command:
-              - --app=build/rn0.66.ipa
-              - --farm=bb
-              - --device=IOS_15|IOS_16
-              - --fail-fast
-              - --no-tunnel
-              - --aws-public-ip
-          test-collector#v1.10.2:
-            files: "reports/TEST-*.xml"
-            format: "junit"
-            branch: "^main|next$$"
-            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
-        env:
-          SKIP_NAVIGATION_SCENARIOS: "true"
-        concurrency: 25
-        concurrency_group: "bitbar"
-        concurrency_method: eager
-
-      - label: ":bitbar: :ios: RN 0.67 iOS end-to-end tests"
-        depends_on: "rn-0-67-ipa"
+      - label: ":bitbar: :ios: RN 0.68 (hermes) iOS end-to-end tests"
+        depends_on: "rn-0-68-hermes-ipa"
         timeout_in_minutes: 40
         plugins:
           artifacts#v1.5.0:
-            download: "build/rn0.67.ipa"
+            download: "build/rn0.68-hermes.ipa"
             upload: ./test/react-native/maze_output/**/*
           docker-compose#v4.12.0:
             pull: react-native-maze-runner
             run: react-native-maze-runner
             service-ports: true
             use-aliases: true
             command:
-              - --app=build/rn0.67.ipa
+              - --app=build/rn0.68-hermes.ipa
               - --farm=bb
               - --device=IOS_15|IOS_16
               - --fail-fast
@@ -279,38 +211,7 @@ steps:
         concurrency: 5
         concurrency_group: "browserstack-app"
         concurrency_method: eager
-      #
-      - label: ":bitbar: :ios: RN 0.68 (hermes) iOS end-to-end tests"
-        depends_on: "rn-0-68-hermes-ipa"
-        timeout_in_minutes: 40
-        plugins:
-          artifacts#v1.5.0:
-            download: "build/rn0.68-hermes.ipa"
-            upload: ./test/react-native/maze_output/**/*
-          docker-compose#v4.12.0:
-            pull: react-native-maze-runner
-            run: react-native-maze-runner
-            service-ports: true
-            use-aliases: true
-            command:
-              - --app=build/rn0.68-hermes.ipa
-              - --farm=bb
-              - --device=IOS_15|IOS_16
-              - --fail-fast
-              - --no-tunnel
-              - --aws-public-ip
-          test-collector#v1.10.2:
-            files: "reports/TEST-*.xml"
-            format: "junit"
-            branch: "^main|next$$"
-            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
-        env:
-          SKIP_NAVIGATION_SCENARIOS: "true"
-          HERMES: "true"
-        concurrency: 5
-        concurrency_group: "browserstack-app"
-        concurrency_method: eager
-
+
       - label: ":bitbar: :mac: RN {{matrix}} iOS (Old Arch) end-to-end tests"
         depends_on: "build-react-native-ios-fixture-old-arch-full"
         timeout_in_minutes: 40
@@ -344,9 +245,7 @@ steps:
         concurrency_group: "bitbar"
         concurrency_method: eager
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"
@@ -384,9 +283,7 @@ steps:
         concurrency_group: "bitbar"
         concurrency_method: eager
         matrix:
-          - "0.71"
           - "0.72"
-          - "0.73"
           - "0.74"
           - "0.75"
           - "0.76"

test/react-native/TESTING.md

@@ -26,7 +26,7 @@ From the root directory run `node ./scripts/generate-react-native-fixture.js`, s
 - `RN_VERSION`
 - `REGISTRY_URL`
 - `NOTIFIER_VERSION`
-- `RCT_NEW_ARCH_ENABLED` (optional - set to 1 to build a fixture with New Architecture enabled)
+- `RCT_NEW_ARCH_ENABLED` - 1 (new architecture) or 0 (old architecture)
 - `REACT_NATIVE_NAVIGATION` (optional) - set to 1 to build a react-native-navigation test fixture
 - `BUILD_ANDROID` (optional) - set to 1 to build an Android APK
 - `BUILD_IOS` (optional) - set to 1 to build an iOS ipa
@@ -108,3 +108,10 @@ particular, these commands need the `BrowserStackLocal` binary (available
     ```
 1. To run all features, omit the final argument.
 1. Maze Runner also supports all options that Cucumber does.  Run `bundle exec maze-runner --help` for full details.
+
+## CI test matrix
+
+End-to-end tests in CI run on both Android and iOS, for both old and new architecture (where supported), across a range of React Native versions. When a new version of React Native is released, the CI test matrix should be updated to test against the following React Native versions:
+
+- The latest 3 versions of React Native
+- Even versions of React Native going back to 0.72