Skip to content

Commit

Permalink
Generalize Backwards Compatibility tests so we can test from any vers…
Browse files Browse the repository at this point in the history
…ion to any version

With an issue reported indicating that there are serialization issue between 1.3 and 2.X, making sure that we can reproduce the errors. This new workflow(s) will make sure that we aren't breaking BWC with changes we are adding to 2.X and will give us the flexibility to test certain migration workflows.

Fixing an issue where the BWC tests were not actually building or executing causing the clusters to spin up and then immediately spin down. We will need to invest more energy into running multiple kinds of security plugin specific scenarios through the test system.

Signed-off-by: Peter Nied <[email protected]>
  • Loading branch information
peternied committed Nov 14, 2022
1 parent 93fe633 commit aca655b
Show file tree
Hide file tree
Showing 6 changed files with 204 additions and 96 deletions.
47 changes: 47 additions & 0 deletions .github/actions/create-bwc-build/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: 'Create a backwards compatible ready build'
description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests'

inputs:
plugin-branch:
description: 'The branch of the plugin that should be built, e.g "2.2", "1.x"'
required: true

outputs:
built-version:
description: 'The version of OpenSearch that was associated with this branch'
value: ${{ steps.get-opensearch-version.outputs.built-version }}

runs:
using: "composite"
steps:
# Configure longpath names if on Windows
- name: Enable Longpaths if on Windows
if: ${{ runner.os == 'Windows' }}
run: git config --system core.longpaths true
shell: pwsh

- uses: actions/checkout@v3
with:
repository: opensearch-project/security
ref: ${{ inputs.plugin-branch }}
path: ${{ inputs.plugin-branch }}

- name: Build
uses: gradle/gradle-build-action@v2
with:
arguments: assemble -Dbuild.snapshot=false
build-root-directory: ${{ inputs.plugin-branch }}

# This might not work in windows
- id: get-opensearch-version
run: |
export extractedVersion=$(./gradlew properties -Dbuild.snapshot=false | grep ^version: | awk '{split($0, a, ": ");print a[2]}')
echo "::set-output name=built-version::$extractedVersion"
working-directory: ${{ inputs.plugin-branch }}
shell: bash

- name: Copy current distro into the expected folder
run: |
mkdir -p ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.built-version }}
cp ${{ inputs.plugin-branch }}/build/distributions/opensearch-security-${{ steps.get-opensearch-version.outputs.built-version }}.zip ./bwc-test/src/test/resources/${{ steps.get-opensearch-version.outputs.built-version }}
shell: bash
46 changes: 46 additions & 0 deletions .github/actions/run-bwc-suite/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: 'Create a backwards compatible ready build'
description: 'Checkouts the official version of a the Security plugin and builds it so it can be used for BWC tests'

inputs:
plugin-previous-branch:
description: 'The branch of the plugin that should be built for the previous version, e.g "2.2", "1.x"'
required: true

plugin-next-branch:
description: 'The branch of the plugin that should be built for the next version, e.g "2.3", "main"'
required: true

report-artifact-name:
description: 'The name of the artifacts for this run, e.g. "BWC-2.1-to-2.4-results"'
required: true

runs:
using: "composite"
steps:

- id: build-previous
uses: ./.github/actions/create-bwc-build
with:
plugin-branch: ${{ inputs.plugin-previous-branch }}

- id: build-next
uses: ./.github/actions/create-bwc-build
with:
plugin-branch: ${{ inputs.plugin-next-branch }}

- name: Run BWC tests
uses: gradle/gradle-build-action@v2
with:
arguments: |
bwcTestSuite
-Dtests.security.manager=false
-Dbwc.version.previous=${{ steps.build-previous.outputs.built-version }}
-Dbwc.version.next=${{ steps.build-next.outputs.built-version }} -i
build-root-directory: bwc-test

- uses: actions/upload-artifact@v3
if: always()
with:
name: ${{ inputs.report-artifact-name }}
path: |
./bwc-test/build/reports/
42 changes: 42 additions & 0 deletions .github/workflows/bwc-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Backwards Compability Suite

on: [workflow_dispatch]

jobs:
last-supported-major:
name: Make sure that the last supported major version can mov
runs-on: ubuntu-latest

steps:
- uses: actions/setup-java@v1
with:
java-version: 11

- name: Checkout Security Repo
uses: actions/checkout@v2

- id: build-previous
uses: ./.github/actions/run-bwc-suite
with:
plugin-previous-branch: "1.3"
plugin-next-branch: "2.x"
report-artifact-name: BWC-Last-Supported-Major

next-major:
name: Make sure that the current version is compatible with
runs-on: ubuntu-latest

steps:
- uses: actions/setup-java@v1
with:
java-version: 11

- name: Checkout Security Repo
uses: actions/checkout@v2

- id: build-previous
uses: ./.github/actions/run-bwc-suite
with:
plugin-previous-branch: "2.x"
plugin-next-branch: "main"
report-artifact-name: BWC-Next-Major
34 changes: 19 additions & 15 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,24 +84,28 @@ jobs:
-x spotbugsMain
backward-compatibility:
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
jdk: [11, 17]
platform: ["ubuntu-latest", "windows-latest"]
runs-on: ${{ matrix.platform }}

steps:
- uses: actions/checkout@v2
- uses: actions/setup-java@v1
with:
java-version: 11
- run: ./gradlew clean build -Dbuild.snapshot=false -x test -x integrationTest
- run: |
echo "Running backwards compatibility tests ..."
security_plugin_version_no_snapshot=$(./gradlew properties -q | grep -E '^version:' | awk '{print $2}' | sed 's/-SNAPSHOT//g')
cp -r build/ ./bwc-test/
mkdir ./bwc-test/src/test/resources/security_plugin_version_no_snapshot
cp build/distributions/opensearch-security-${security_plugin_version_no_snapshot}.zip ./bwc-test/src/test/resources/${security_plugin_version_no_snapshot}
mkdir bwc-test/src/test/resources/2.4.0.0
wget https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.4.0/latest/linux/x64/tar/builds/opensearch/plugins/opensearch-security-2.4.0.0.zip
mv opensearch-security-2.4.0.0.zip bwc-test/src/test/resources/2.4.0.0/
cd bwc-test/
./gradlew bwcTestSuite -Dtests.security.manager=false
java-version: ${{ matrix.jdk }}

- name: Checkout Security Repo
uses: actions/checkout@v2

- id: build-previous
uses: ./.github/actions/run-bwc-suite
with:
plugin-previous-branch: "2.4"
plugin-next-branch: "main"
report-artifact-name: bwc-${{ matrix.platform }}-jdk${{ matrix.jdk }}

code-ql:
runs-on: ubuntu-latest
Expand Down
81 changes: 23 additions & 58 deletions bwc-test/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,7 @@ apply plugin: 'java'

apply plugin: 'opensearch.testclusters'

compileTestJava.enabled = false

ext {
projectSubstitutions = [:]
licenseFile = rootProject.file('LICENSE.TXT')
noticeFile = rootProject.file('NOTICE')
}
Expand Down Expand Up @@ -70,19 +67,37 @@ repositories {
}

dependencies {
testImplementation 'com.google.guava:guava:30.0-jre'
testImplementation "org.opensearch.test:framework:${opensearch_version}"
testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1'
}

String bwcVersion = "2.4.0.0";
loggerUsageCheck.enabled = false
testingConventions.enabled = false
validateNebulaPom.enabled = false

String previousVersion = System.getProperty("bwc.version.previous", "2.4.0.0")
String nextVersion = System.getProperty("bwc.version.next", "3.0.0.0")

String bwcVersion = previousVersion
String baseName = "securityBwcCluster"
String bwcFilePath = "src/test/resources/"
String projectVersion = "3.0.0.0"
String projectVersion = nextVersion

def previousVersionMatcher = previousVersion =~ /(.+?)(\.\d+)$/
previousVersionMatcher.find()
String previousOpenSearch = previousVersionMatcher.group(1)

def nextVersionMatcher = nextVersion =~ /(.+?)(\.\d+)$/
nextVersionMatcher.find()
String nextOpenSearch = nextVersionMatcher.group(1)


2.times {i ->
testClusters {
"${baseName}$i" {
testDistribution = "ARCHIVE"
versions = ["2.4.0","3.0.0"]
versions = [previousOpenSearch, nextOpenSearch]
numberOfNodes = 3
plugin(provider(new Callable<RegularFile>() {
@Override
Expand Down Expand Up @@ -148,24 +163,7 @@ List<Provider<RegularFile>> plugins = [
// Creates a test cluster with 3 nodes of the old version.
2.times {i ->
task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
useCluster testClusters."${baseName}$i"
if (System.getProperty("mixedCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'old'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -178,23 +176,11 @@ List<Provider<RegularFile>> plugins = [
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#oldVersionClusterTask0"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("mixedCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAMixedCluster")
}
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'first'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -206,18 +192,11 @@ task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#mixedClusterTask"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'second'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -229,18 +208,11 @@ task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTas
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
if (System.getProperty("rollingUpgradeCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInARollingUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'third'
systemProperty 'tests.plugin_bwc_version', bwcVersion
Expand All @@ -251,28 +223,21 @@ task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask)
// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
tasks.register("${baseName}#fullRestartClusterTask", StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
dependsOn "${baseName}#oldVersionClusterTask1"
useCluster testClusters."${baseName}1"
doFirst {
testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
if (System.getProperty("fullRestartCluster") != null) {
filter {
includeTest("org.opensearch.security.bwc.SecurityBackwardsCompatibilityIT", "testPluginUpgradeInAnUpgradedCluster")
}
}
systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
systemProperty 'tests.plugin_bwc_version', bwcVersion
systemProperty 'tests.rest.bwcsuite_round', 'first'
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}

// A bwc test suite which runs all the bwc tasks combined.
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
exclude '**/*Test*'
exclude '**/*Sanity*'
exclude '**/**' // Do not run any tests as part of this aggregate task
dependsOn tasks.named("${baseName}#mixedClusterTask")
dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
dependsOn tasks.named("${baseName}#fullRestartClusterTask")
Expand Down
Loading

0 comments on commit aca655b

Please sign in to comment.