From d2efd08358fc07c92f3eada7a48c7158c4d0bec4 Mon Sep 17 00:00:00 2001 From: Peter Nied Date: Thu, 10 Nov 2022 20:59:18 +0000 Subject: [PATCH] Generalize Backwards Compatibility tests so we can test from any version to any version With an issue reported indicating that there are serialization issue between 1.3 and 2.X, making sure that we can reproduce the errors. This new workflow(s) will make sure that we aren't breaking BWC with changes we are adding to 2.X and will give us the flexibility to test certain migration workflows. Fixing an issue where the BWC tests were not actually building or executing causing the clusters to spin up and then immediately spin down. We will need to invest more energy into running multiple kinds of security plugin specific scenarios through the test system. Signed-off-by: Peter Nied --- .github/actions/create-bwc-build/action.yaml | 47 +++++++++++ .github/actions/run-bwc-suite/action.yaml | 46 +++++++++++ .github/workflows/bwc-tests.yml | 42 ++++++++++ .github/workflows/ci.yml | 34 ++++---- bwc-test/build.gradle | 81 ++++++------------- .../SecurityBackwardsCompatibilityIT.java | 50 ++++++------ 6 files changed, 204 insertions(+), 96 deletions(-) create mode 100644 .github/actions/create-bwc-build/action.yaml create mode 100644 .github/actions/run-bwc-suite/action.yaml create mode 100644 .github/workflows/bwc-tests.yml diff --git a/.github/actions/create-bwc-build/action.yaml b/.github/actions/create-bwc-build/action.yaml new file mode 100644 index 0000000000..e75e20f05c --- /dev/null +++ b/.github/actions/create-bwc-build/action.yaml @@ -0,0 +1,47 @@ +name: 'Create a backwards compatible ready build' +description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests' + +inputs: + plugin-branch: + description: 'The branch of the plugin that should be built, e.g "2.2", "1.x"' + required: true + +outputs: + built-version: + description: 'The version of OpenSearch that was associated with this branch' + value: ${{ steps.get-opensearch-version.outputs.built-version }} + +runs: + using: "composite" + steps: + # Configure longpath names if on Windows + - name: Enable Longpaths if on Windows + if: ${{ runner.os == 'Windows' }} + run: git config --system core.longpaths true + shell: pwsh + + - uses: actions/checkout@v3 + with: + repository: opensearch-project/security + ref: ${{ inputs.plugin-branch }} + path: ${{ inputs.plugin-branch }} + + - name: Build + uses: gradle/gradle-build-action@v2 + with: + arguments: assemble -Dbuild.snapshot=false + build-root-directory: ${{ inputs.plugin-branch }} + + # This might not work in windows + - id: get-opensearch-version + run: | + export extractedVersion=$(./gradlew properties -Dbuild.snapshot=false | grep ^version: | awk '{split($0, a, ": ");print a[2]}') + echo "::set-output name=built-version::$extractedVersion" + working-directory: ${{ inputs.plugin-branch }} + shell: bash + + - name: Copy current distro into the expected folder + run: | + mkdir -p ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.built-version }} + cp ${{ inputs.plugin-branch }}/build/distributions/opensearch-security-${{ steps.get-opensearch-version.outputs.built-version }}.zip ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.built-version }} + shell: bash diff --git a/.github/actions/run-bwc-suite/action.yaml b/.github/actions/run-bwc-suite/action.yaml new file mode 100644 index 0000000000..ff6055e581 --- /dev/null +++ b/.github/actions/run-bwc-suite/action.yaml @@ -0,0 +1,46 @@ +name: 'Create a backwards compatible ready build' +description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests' + +inputs: + plugin-previous-branch: + description: 'The branch of the plugin that should be built for the previous version, e.g "2.2", "1.x"' + required: true + + plugin-next-branch: + description: 'The branch of the plugin that should be built for the next version, e.g "2.3", "main"' + required: true + + report-artifact-name: + description: 'The name of the artifacts for this run, e.g. "BWC-2.1-to-2.4-results"' + required: true + +runs: + using: "composite" + steps: + + - id: build-previous + uses: ./.github/actions/create-bwc-build + with: + plugin-branch: ${{ inputs.plugin-previous-branch }} + + - id: build-next + uses: ./.github/actions/create-bwc-build + with: + plugin-branch: ${{ inputs.plugin-next-branch }} + + - name: Run BWC tests + uses: gradle/gradle-build-action@v2 + with: + arguments: | + bwcTestSuite + -Dtests.security.manager=false + -Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }} + -Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i + build-root-directory: bwc-test + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ inputs.report-artifact-name }} + path: | + ./bwc-test/build/reports/ diff --git a/.github/workflows/bwc-tests.yml b/.github/workflows/bwc-tests.yml new file mode 100644 index 0000000000..a330eb3de5 --- /dev/null +++ b/.github/workflows/bwc-tests.yml @@ -0,0 +1,42 @@ +name: Backwards Compability Suite + +on: [workflow_dispatch] + +jobs: + last-supported-major: + name: Make sure that the last supported major version can mov + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Checkout Security Repo + uses: actions/checkout@v2 + + - id: build-previous + uses: ./.github/actions/run-bwc-suite + with: + plugin-previous-branch: "1.3" + plugin-next-branch: "2.x" + report-artifact-name: BWC-Last-Supported-Major + + next-major: + name: Make sure that the current version is compatible with + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-java@v1 + with: + java-version: 11 + + - name: Checkout Security Repo + uses: actions/checkout@v2 + + - id: build-previous + uses: ./.github/actions/run-bwc-suite + with: + plugin-previous-branch: "2.x" + plugin-next-branch: "main" + report-artifact-name: BWC-Next-Major diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e7fc78987..4284884f74 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,24 +84,28 @@ jobs: -x spotbugsMain backward-compatibility: - runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + jdk: [11, 17] + platform: ["ubuntu-latest", "windows-latest"] + runs-on: ${{ matrix.platform }} + steps: - - uses: actions/checkout@v2 - uses: actions/setup-java@v1 with: - java-version: 11 - - run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest - - run: | - echo "Running backwards compatibility tests ..." - security_plugin_version_no_snapshot=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}' | sed 's/-SNAPSHOT//g') - cp -r build/ ./bwc-test/ - mkdir ./bwc-test/src/test/resources/security_plugin_version_no_snapshot - cp build/distributions/opensearch-security-${security_plugin_version_no_snapshot}.zip ./bwc-test/src/test/resources/${security_plugin_version_no_snapshot} - mkdir bwc-test/src/test/resources/2.4.0.0 - wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-2.4.0.0.zip - mv opensearch-security-2.4.0.0.zip bwc-test/src/test/resources/2.4.0.0/ - cd bwc-test/ - ./gradlew bwcTestSuite -Dtests.security.manager=false + java-version: ${{ matrix.jdk }} + + - name: Checkout Security Repo + uses: actions/checkout@v2 + + - id: build-previous + uses: ./.github/actions/run-bwc-suite + with: + plugin-previous-branch: "2.4" + plugin-next-branch: "main" + report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }} code-ql: runs-on: ubuntu-latest diff --git a/bwc-test/build.gradle b/bwc-test/build.gradle index 9badfd1c85..07585de5d9 100644 --- a/bwc-test/build.gradle +++ b/bwc-test/build.gradle @@ -37,10 +37,7 @@ apply plugin: 'java' apply plugin: 'opensearch.testclusters' -compileTestJava.enabled = false - ext { - projectSubstitutions = [:] licenseFile = rootProject.file('LICENSE.TXT') noticeFile = rootProject.file('NOTICE') } @@ -70,19 +67,37 @@ repositories { } dependencies { + testImplementation 'com.google.guava:guava:30.0-jre' testImplementation "org.opensearch.test:framework:${opensearch_version}" + testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' } -String bwcVersion = "2.4.0.0"; +loggerUsageCheck.enabled = false +testingConventions.enabled = false +validateNebulaPom.enabled = false + +String previousVersion = System.getProperty("bwc.version.previous", "2.4.0.0") +String nextVersion = System.getProperty("bwc.version.next", "3.0.0.0") + +String bwcVersion = previousVersion String baseName = "securityBwcCluster" String bwcFilePath = "src/test/resources/" -String projectVersion = "3.0.0.0" +String projectVersion = nextVersion + +def previousVersionMatcher = previousVersion =~ /(.+?)(\.\d+)$/ +previousVersionMatcher.find() +String previousOpenSearch = previousVersionMatcher.group(1) + +def nextVersionMatcher = nextVersion =~ /(.+?)(\.\d+)$/ +nextVersionMatcher.find() +String nextOpenSearch = nextVersionMatcher.group(1) + 2.times {i -> testClusters { "${baseName}$i" { testDistribution = "ARCHIVE" - versions = ["2.4.0","3.0.0"] + versions = [previousOpenSearch, nextOpenSearch] numberOfNodes = 3 plugin(provider(new Callable() { @Override @@ -148,24 +163,7 @@ List> plugins = [ // Creates a test cluster with 3 nodes of the old version. 2.times {i -> task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*Sanity*' useCluster testClusters."${baseName}$i" - if (System.getProperty("mixedCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster") - } - } - if (System.getProperty("rollingUpgradeCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") - } - } - if (System.getProperty("fullRestartCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster") - } - } systemProperty 'tests.rest.bwcsuite', 'old_cluster' systemProperty 'tests.rest.bwcsuite_round', 'old' systemProperty 'tests.plugin_bwc_version', bwcVersion @@ -178,23 +176,11 @@ List> plugins = [ // This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node. // This is also used as a one third upgraded cluster for a rolling upgrade. task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*Sanity*' dependsOn "${baseName}#oldVersionClusterTask0" useCluster testClusters."${baseName}0" doFirst { testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) } - if (System.getProperty("mixedCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster") - } - } - if (System.getProperty("rollingUpgradeCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") - } - } systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'first' systemProperty 'tests.plugin_bwc_version', bwcVersion @@ -206,18 +192,11 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) { // This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes. // This is used for rolling upgrade. task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*Sanity*' dependsOn "${baseName}#mixedClusterTask" useCluster testClusters."${baseName}0" doFirst { testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) } - if (System.getProperty("rollingUpgradeCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") - } - } systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'second' systemProperty 'tests.plugin_bwc_version', bwcVersion @@ -229,18 +208,11 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas // This results in a fully upgraded cluster. // This is used for rolling upgrade. task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*Sanity*' dependsOn "${baseName}#twoThirdsUpgradedClusterTask" useCluster testClusters."${baseName}0" doFirst { testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins) } - if (System.getProperty("rollingUpgradeCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster") - } - } systemProperty 'tests.rest.bwcsuite', 'mixed_cluster' systemProperty 'tests.rest.bwcsuite_round', 'third' systemProperty 'tests.plugin_bwc_version', bwcVersion @@ -251,28 +223,21 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) // Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version // at the same time resulting in a fully upgraded cluster. tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*Sanity*' dependsOn "${baseName}#oldVersionClusterTask1" useCluster testClusters."${baseName}1" doFirst { testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins) } - if (System.getProperty("fullRestartCluster") != null) { - filter { - includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster") - } - } systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster' systemProperty 'tests.plugin_bwc_version', bwcVersion + systemProperty 'tests.rest.bwcsuite_round', 'first' nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}") } // A bwc test suite which runs all the bwc tasks combined. task bwcTestSuite(type: StandaloneRestIntegTestTask) { - exclude '**/*Test*' - exclude '**/*Sanity*' + exclude '**/**' // Do not run any tests as part of this aggregate task dependsOn tasks.named("${baseName}#mixedClusterTask") dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask") dependsOn tasks.named("${baseName}#fullRestartClusterTask") diff --git a/bwc-test/src/test/java/SecurityBackwardsCompatibilityIT.java b/bwc-test/src/test/java/SecurityBackwardsCompatibilityIT.java index 273de3f6b2..1afc1b88d5 100644 --- a/bwc-test/src/test/java/SecurityBackwardsCompatibilityIT.java +++ b/bwc-test/src/test/java/SecurityBackwardsCompatibilityIT.java @@ -12,18 +12,32 @@ import java.util.Set; import java.util.stream.Collectors; +import org.junit.Assume; import org.junit.Assert; +import org.junit.Before; import org.opensearch.client.Response; import org.opensearch.common.settings.Settings; import org.opensearch.rest.RestStatus; import org.opensearch.test.rest.OpenSearchRestTestCase; +import org.opensearch.Version; import com.google.common.collect.ImmutableMap; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.hasItem; + public class SecurityBackwardsCompatibilityIT extends OpenSearchRestTestCase { - private static final ClusterType CLUSTER_TYPE = ClusterType.parse(System.getProperty("tests.rest.bwcsuite")); - private static final String CLUSTER_NAME = System.getProperty("tests.clustername"); + private ClusterType CLUSTER_TYPE; + private String CLUSTER_NAME; + + @Before + private void testSetup() { + final String bwcsuiteString = System.getProperty("tests.rest.bwcsuite"); + Assume.assumeTrue("Test cannot be run outside the BWC gradle task 'bwcTestSuite' or its dependent tasks", bwcsuiteString != null); + CLUSTER_TYPE = ClusterType.parse(bwcsuiteString); + CLUSTER_NAME = System.getProperty("tests.clustername"); + } @Override protected final boolean preserveIndicesUponCompletion() { @@ -52,16 +66,9 @@ protected final Settings restClientSettings() { .build(); } - public void testPluginUpgradeInAMixedCluster() throws Exception { - assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-0/plugins"); - } - - public void testPluginUpgradeInAnUpgradedCluster() throws Exception { - assertPluginUpgrade("_nodes/plugins"); - } - - public void testPluginUpgradeInARollingUpgradedCluster() throws Exception { + public void testBasicBackwardsCompatibility() throws Exception { String round = System.getProperty("tests.rest.bwcsuite_round"); + if (round.equals("first") || round.equals("old")) { assertPluginUpgrade("_nodes/" + CLUSTER_NAME + "-0/plugins"); } else if (round.equals("second")) { @@ -95,19 +102,16 @@ private void assertPluginUpgrade(String uri) throws Exception { Map> responseMap = (Map>) getAsMap(uri).get("nodes"); for (Map response : responseMap.values()) { List> plugins = (List>) response.get("plugins"); - Set pluginNames = plugins.stream().map(map -> map.get("name")).collect(Collectors.toSet()); - switch (CLUSTER_TYPE) { - case OLD: - Assert.assertTrue(pluginNames.contains("opendistro_security")); - break; - case MIXED: - Assert.assertTrue(pluginNames.contains("opensearch-security")); - break; - case UPGRADED: - Assert.assertTrue(pluginNames.contains("opendistro_security")); - break; + Set pluginNames = plugins.stream().map(map -> (String) map.get("name")).collect(Collectors.toSet()); + + final Version minNodeVersion = this.minimumNodeVersion(); + + if (minNodeVersion.major <= 1) { + assertThat(pluginNames, hasItem("opensearch_security")); + } else { + assertThat(pluginNames, hasItem("opensearch-security")); } - break; + } } }