Generalize Backwards Compatibility tests so we can test from any version to any version

.github/actions/create-bwc-build/action.yaml

@@ -0,0 +1,47 @@
+name: 'Create a backwards compatible ready build'
+description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests'
+
+inputs:
+  plugin-branch:
+    description: 'The branch of the plugin that should be built, e.g "2.2", "1.x"'
+    required: true
+
+outputs:
+  built-version:
+    description: 'The version of OpenSearch that was associated with this branch'
+    value: ${{ steps.get-opensearch-version.outputs.built-version }}
+
+runs:
+  using: "composite"
+  steps:
+      # Configure longpath names if on Windows
+    - name: Enable Longpaths if on Windows
+      if: ${{ runner.os == 'Windows' }}
+      run: git config --system core.longpaths true
+      shell: pwsh
+
+    - uses: actions/checkout@v3
+      with:
+        repository: opensearch-project/security
+        ref: ${{ inputs.plugin-branch }}
+        path: ${{ inputs.plugin-branch }}
+
+    - name: Build
+      uses: gradle/gradle-build-action@v2
+      with:
+        arguments: assemble -Dbuild.snapshot=false
+        build-root-directory: ${{ inputs.plugin-branch }}
+
+    # This might not work in windows
+    - id: get-opensearch-version
+      run: |
+        export extractedVersion=$(./gradlew properties -Dbuild.snapshot=false | grep ^version: | awk '{split($0, a, ": ");print a[2]}')
+        echo "::set-output name=built-version::$extractedVersion"
+      working-directory: ${{ inputs.plugin-branch }}
+      shell: bash
+
+    - name: Copy current distro into the expected folder
+      run: |
+        mkdir -p ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.built-version }}
+        cp ${{ inputs.plugin-branch }}/build/distributions/opensearch-security-${{ steps.get-opensearch-version.outputs.built-version }}.zip ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.built-version }}
+      shell: bash

.github/actions/run-bwc-suite/action.yaml

@@ -0,0 +1,46 @@
+name: 'Create a backwards compatible ready build'
+description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests'
+
+inputs:
+  plugin-previous-branch:
+    description: 'The branch of the plugin that should be built for the previous version, e.g "2.2", "1.x"'
+    required: true
+
+  plugin-next-branch:
+    description: 'The branch of the plugin that should be built for the next version, e.g "2.3", "main"'
+    required: true
+
+  report-artifact-name:
+    description: 'The name of the artifacts for this run, e.g. "BWC-2.1-to-2.4-results"'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+
+    - id: build-previous
+      uses: ./.github/actions/create-bwc-build
+      with:
+        plugin-branch: ${{ inputs.plugin-previous-branch }}
+
+    - id: build-next
+      uses: ./.github/actions/create-bwc-build
+      with:
+        plugin-branch: ${{ inputs.plugin-next-branch }}
+
+    - name: Run BWC tests
+      uses: gradle/gradle-build-action@v2
+      with:
+        arguments: |
+          bwcTestSuite
+          -Dtests.security.manager=false
+          -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }}
+          -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i
+        build-root-directory: bwc-test
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: ${{ inputs.report-artifact-name }}
+        path: |
+          ./bwc-test/build/reports/

.github/workflows/bwc-tests.yml

@@ -0,0 +1,42 @@
+name: Backwards Compability Suite
+
+on: [workflow_dispatch]
+
+jobs:
+  last-supported-major:
+    name: Make sure that the last supported major version can mov
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/setup-java@v1
+      with:
+        java-version: 11
+
+    - name: Checkout Security Repo
+      uses: actions/checkout@v2
+
+    - id: build-previous
+      uses: ./.github/actions/run-bwc-suite
+      with:
+        plugin-previous-branch: "1.3"
+        plugin-next-branch: "2.x"
+        report-artifact-name: BWC-Last-Supported-Major
+
+  next-major:
+    name: Make sure that the current version is compatible with
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/setup-java@v1
+      with:
+        java-version: 11
+
+    - name: Checkout Security Repo
+      uses: actions/checkout@v2
+
+    - id: build-previous
+      uses: ./.github/actions/run-bwc-suite
+      with:
+        plugin-previous-branch: "2.x"
+        plugin-next-branch: "main"
+        report-artifact-name: BWC-Next-Major

.github/workflows/ci.yml

@@ -84,24 +84,28 @@ jobs:
           -x spotbugsMain
 
   backward-compatibility:
-    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        jdk: [11, 17]
+        platform: ["ubuntu-latest", "windows-latest"]
+    runs-on: ${{ matrix.platform }}
+
     steps:
-    - uses: actions/checkout@v2
     - uses: actions/setup-java@v1
       with:
-        java-version: 11
-    - run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest
-    - run: |
-        echo "Running backwards compatibility tests ..."
-        security_plugin_version_no_snapshot=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}' | sed 's/-SNAPSHOT//g')
-        cp -r build/ ./bwc-test/
-        mkdir ./bwc-test/src/test/resources/security_plugin_version_no_snapshot
-        cp build/distributions/opensearch-security-${security_plugin_version_no_snapshot}.zip ./bwc-test/src/test/resources/${security_plugin_version_no_snapshot}
-        mkdir bwc-test/src/test/resources/2.4.0.0
-        wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-2.4.0.0.zip
-        mv opensearch-security-2.4.0.0.zip bwc-test/src/test/resources/2.4.0.0/
-        cd bwc-test/
-        ./gradlew bwcTestSuite -Dtests.security.manager=false
+        java-version: ${{ matrix.jdk }}
+
+    - name: Checkout Security Repo
+      uses: actions/checkout@v2
+
+    - id: build-previous
+      uses: ./.github/actions/run-bwc-suite
+      with:
+        plugin-previous-branch: "2.4"
+        plugin-next-branch: "main"
+        report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }}
 
   code-ql:
     runs-on: ubuntu-latest

bwc-test/build.gradle

@@ -37,10 +37,7 @@ apply plugin: 'java'
 
 apply plugin: 'opensearch.testclusters'
 
-compileTestJava.enabled = false
-
 ext {
-    projectSubstitutions = [:]
     licenseFile = rootProject.file('LICENSE.TXT')
     noticeFile = rootProject.file('NOTICE')
 }
@@ -70,19 +67,37 @@ repositories {
 }
 
 dependencies {
+    testImplementation 'com.google.guava:guava:30.0-jre'
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
+    testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1'
 }
 
-String bwcVersion = "2.4.0.0";
+loggerUsageCheck.enabled = false
+testingConventions.enabled = false
+validateNebulaPom.enabled = false
+
+String previousVersion = System.getProperty("bwc.version.previous", "2.4.0.0")
+String nextVersion = System.getProperty("bwc.version.next", "3.0.0.0")
+
+String bwcVersion = previousVersion
 String baseName = "securityBwcCluster"
 String bwcFilePath = "src/test/resources/"
-String projectVersion = "3.0.0.0"
+String projectVersion = nextVersion
+
+def previousVersionMatcher =  previousVersion =~ /(.+?)(\.\d+)$/
+previousVersionMatcher.find()
+String previousOpenSearch  = previousVersionMatcher.group(1)
+
+def nextVersionMatcher =  nextVersion =~ /(.+?)(\.\d+)$/
+nextVersionMatcher.find()
+String nextOpenSearch  = nextVersionMatcher.group(1)
+
 
 2.times {i ->
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["2.4.0","3.0.0"]
+            versions = [previousOpenSearch, nextOpenSearch]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>() {
                 @Override
@@ -148,24 +163,7 @@ List<Provider<RegularFile>> plugins = [
 // Creates a test cluster with 3 nodes of the old version.
 2.times {i ->
     task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
-        exclude '**/*Test*'
-        exclude '**/*Sanity*'
         useCluster testClusters."${baseName}$i"
-        if (System.getProperty("mixedCluster") != null) {
-            filter {
-                includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
-            }
-        }
-        if (System.getProperty("rollingUpgradeCluster") != null) {
-            filter {
-                includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
-            }
-        }
-        if (System.getProperty("fullRestartCluster") != null) {
-            filter {
-                includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
-            }
-        }
         systemProperty 'tests.rest.bwcsuite', 'old_cluster'
         systemProperty 'tests.rest.bwcsuite_round', 'old'
         systemProperty 'tests.plugin_bwc_version', bwcVersion
@@ -178,23 +176,11 @@ List<Provider<RegularFile>> plugins = [
 // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
 // This is also used as a one third upgraded cluster for a rolling upgrade.
 task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
-    exclude '**/*Test*'
-    exclude '**/*Sanity*'
     dependsOn "${baseName}#oldVersionClusterTask0"
     useCluster testClusters."${baseName}0"
     doFirst {
         testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
     }
-    if (System.getProperty("mixedCluster") != null) {
-        filter {
-            includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
-        }
-    }
-    if (System.getProperty("rollingUpgradeCluster") != null) {
-        filter {
-            includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
-        }
-    }
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'first'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
@@ -206,18 +192,11 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
 // This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
 // This is used for rolling upgrade.
 task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
-    exclude '**/*Test*'
-    exclude '**/*Sanity*'
     dependsOn "${baseName}#mixedClusterTask"
     useCluster testClusters."${baseName}0"
     doFirst {
         testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
     }
-    if (System.getProperty("rollingUpgradeCluster") != null) {
-        filter {
-            includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
-        }
-    }
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'second'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
@@ -229,18 +208,11 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
 // This results in a fully upgraded cluster.
 // This is used for rolling upgrade.
 task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
-    exclude '**/*Test*'
-    exclude '**/*Sanity*'
     dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
     useCluster testClusters."${baseName}0"
     doFirst {
         testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
     }
-    if (System.getProperty("rollingUpgradeCluster") != null) {
-        filter {
-            includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
-        }
-    }
     systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
     systemProperty 'tests.rest.bwcsuite_round', 'third'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
@@ -251,28 +223,21 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
 // Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
 // at the same time resulting in a fully upgraded cluster.
 tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) {
-    exclude '**/*Test*'
-    exclude '**/*Sanity*'
     dependsOn "${baseName}#oldVersionClusterTask1"
     useCluster testClusters."${baseName}1"
     doFirst {
         testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
     }
-    if (System.getProperty("fullRestartCluster") != null) {
-        filter {
-            includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
-        }
-    }
     systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
     systemProperty 'tests.plugin_bwc_version', bwcVersion
+    systemProperty 'tests.rest.bwcsuite_round', 'first'
     nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
     nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
 }
 
 // A bwc test suite which runs all the bwc tasks combined.
 task bwcTestSuite(type: StandaloneRestIntegTestTask) {
-    exclude '**/*Test*'
-    exclude '**/*Sanity*'
+    exclude '**/**' // Do not run any tests as part of this aggregate task
     dependsOn tasks.named("${baseName}#mixedClusterTask")
     dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
     dependsOn tasks.named("${baseName}#fullRestartClusterTask")