From c81c2b0469cd6788877a4f1324999c3804ca3c5b Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Wed, 27 Mar 2024 09:32:45 -0700 Subject: [PATCH 1/4] Add jenkins benchmark job for endpoint Signed-off-by: Divya Madala --- .../benchmark-test-endpoint.jenkinsfile | 138 ++++++++++++++++++ .../TestRunBenchmarkTestEndpoint.groovy | 127 ++++++++++++++++ .../benchmark-test-endpoint.jenkinsfile.txt | 34 +++++ 3 files changed, 299 insertions(+) create mode 100644 jenkins/opensearch/benchmark-test-endpoint.jenkinsfile create mode 100644 tests/jenkins/TestRunBenchmarkTestEndpoint.groovy create mode 100644 tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt diff --git a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile new file mode 100644 index 0000000000..77f3042e4e --- /dev/null +++ b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile @@ -0,0 +1,138 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([ + + $class: 'GitSCMSource', + remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', +])) + +pipeline { + agent none + options { + timeout(time: 24, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '30')) + } + environment { + AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test' + JOB_NAME = 'benchmark-test' + } +parameters { + string( + name: 'CLUSTER_ENDPOINT', + description: 'Provide an endpoint to a cluster for running benchmark tests against it.', + trim: true + ) + booleanParam( + name: 'CLUSTER_WITH_SECURITY', + description: 'Mention if the cluster is secured or insecured.', + defaultValue: false, + ) + string( + + name: 'TEST_WORKLOAD', + description: 'The workload name from OpenSearch Benchmark Workloads.', + defaultValue: 'nyc_taxis', + trim: true + ) + string( + name: 'ADDITIONAL_CONFIG', + description: 'Additional opensearch.yml config parameters passed as JSON. e.g., `opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT`', + trim: true + ) + string( + name: 'USER_TAGS', + description: 'Attach arbitrary text to the meta-data of each benchmark metric record, without any spaces. e.g., `run-type:adhoc,segrep:enabled,arch:x64`. ', + trim: true + ) + string( + name: 'WORKLOAD_PARAMS', + description: 'With this parameter you can inject variables into workloads. Use json type. e.g., `{"number_of_replicas":"1","number_of_shards":"5"}`', + trim: true + ) + string( + name: 'TEST_PROCEDURE', + description: 'Defines a test procedure to use. e.g., `append-no-conflicts,significant-text`', + trim: true + ) + string( + name: 'EXCLUDE_TASKS', + description: 'Defines a comma-separated list of test procedure tasks not to run. Default runs all. e.g., `type:search,delete-index`', + trim: true + ) + string( + name: 'INCLUDE_TASKS', + description: 'Defines a comma-separated list of test procedure tasks to run. Default runs all. e.g., `type:search,delete-index`', + trim: true + ) + booleanParam( + name: 'CAPTURE_NODE_STAT', + description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.', + defaultValue: false + ) + booleanParam( + name: 'CAPTURE_SEGMENT_REPLICATION_STAT', + description: 'Enable opensearch-benchmark segment-replication-stats telemetry to capture metrics such as replication lag.', + defaultValue: false + ) + string( + name: 'TELEMETRY_PARAMS', + description: 'Allows to set parameters for telemetry devices. Use json type. e.g.,{"node-stats-include-indices":"true","node-stats-include-indices-metrics":"segments"}', + trim: true + ) + } + + stages { + stage('validate-and-set-parameters') { + agent { label AGENT_LABEL } + steps { + script { + if (CLUSTER_ENDPOINT == '') { + currentBuild.result = 'ABORTED' + error("Benchmark Tests failed to start. Provide CLUSTER_ENDPOINT to run tests") + } + } + } + post { + always { + postCleanup() + } + } + } + stage('benchmark-test-with-cluster') { + agent { label AGENT_LABEL } + steps { + script { + echo "CLUSTER_ENDPOINT: ${CLUSTER_ENDPOINT}" + echo "security-enabled: ${CLUSTER_WITH_SECURITY}" + + runBenchmarkTestScript( + endpoint: CLUSTER_ENDPOINT, + insecure: !CLUSTER_WITH_SECURITY, + workload: TEST_WORKLOAD, + userTag: USER_TAGS.isEmpty() ? "security-enabled:${CLUSTER_WITH_SECURITY}" : "${USER_TAGS},security-enabled:${CLUSTER_WITH_SECURITY}", + workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, + additionalConfig: ADDITIONAL_CONFIG, + captureNodeStat: CAPTURE_NODE_STAT, + captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT, + telemetryParams: TELEMETRY_PARAMS + ) + } + post { + always { + postCleanup() + } + } + } + } + } +} diff --git a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy new file mode 100644 index 0000000000..4a1a463524 --- /dev/null +++ b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy @@ -0,0 +1,127 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.CoreMatchers.hasItems +import static org.hamcrest.MatcherAssert.assertThat + +import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library +import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource + +class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ + @Override + @Before + void setUp() { + helper.registerSharedLibrary( + library().name('jenkins') + .defaultVersion('5.11.0') + .allowOverride(true) + .implicit(true) + .targetPath('vars') + .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git')) + .build() + ) + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') + binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1') + binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('CLUSTER_ENDPOINT', 'opensearch-ABCxdfdfhyfk.com') + binding.setVariable('CLUSTER_WITH_SECURITY', true) + binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') + binding.setVariable('GITHUB_USER', 'test_user') + binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('USER_TAGS', 'run-type:test') + binding.setVariable('WORKLOAD_PARAMS', '') + binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts') + binding.setVariable('EXCLUDE_TASKS', '') + binding.setVariable('INCLUDE_TASKS', '') + binding.setVariable('ADDITIONAL_CONFIG', '') + binding.setVariable('JOB_NAME', 'benchmark-test') + binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('STAGE_NAME', 'test_stage') + binding.setVariable('TEST_WORKLOAD', 'nyc-taxis') + binding.setVariable('TELEMETRY_PARAMS', '{"telemetry_setting":"value"}') + + super.setUp() + } + + @Test + public void testRunSecureBenchmarkTestScript_verifyPipeline() { + super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile") + } + + @Test + void testRunSecureBenchmarkTestScript_verifyArtifactDownloads() { + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") + + def curlCommands = getCommandExecutions('sh', 'curl').findAll { + shCommand -> shCommand.contains('curl') + } + + def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll { + shCommand -> shCommand.contains('bucket') + } + + assertThat(s3DownloadCommands.size(), equalTo(1)) + assertThat(s3DownloadCommands, hasItems( + "{file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}".toString() + )) + } + + + @Test + void testRunSecureBenchmarkTestScript_verifyScriptExecutions() { + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItems( + "./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt new file mode 100644 index 0000000000..b25e2401ad --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt @@ -0,0 +1,34 @@ + benchmark-test-endpoint.run() + benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-test-endpoint.library({identifier=jenkins@cluster, retriever=null}) + benchmark-test-endpoint.pipeline(groovy.lang.Closure) + benchmark-test-endpoint.timeout({time=24, unit=HOURS}) + benchmark-test-endpoint.logRotator({daysToKeepStr=30}) + benchmark-test-endpoint.buildDiscarder(null) + benchmark-test-endpoint.echo(Executing on agent [label:none]) + benchmark-test-endpoint.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.echo(CLUSTER_ENDPOINT: opensearch-ABCxdfdfhyfk.com) + benchmark-test-endpoint.echo(security-enabled: true) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, additionalConfig=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) From d810fdea9decedb74b1446d0a1db775e5a04e38d Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Fri, 29 Mar 2024 14:03:20 -0700 Subject: [PATCH 2/4] Add changes and update testcases Signed-off-by: Divya Madala --- .../benchmark-test-endpoint.jenkinsfile | 16 +++------ jenkins/opensearch/benchmark-test.jenkinsfile | 2 +- .../TestRunBenchmarkTestEndpoint.groovy | 27 +++++++++++++-- ...nBenchmarkTestScriptDistributionUrl.groovy | 4 +-- .../TestRunNonSecBenchmarkTestScript.groovy | 4 +-- .../TestRunSecureBenchmarkTestScript.groovy | 6 ++-- ...mark-test-distribution-url.jenkinsfile.txt | 6 ++-- ...ark-test-endpoint-insecure.jenkinsfile.txt | 34 +++++++++++++++++++ ...mark-test-endpoint-secure.jenkinsfile.txt} | 6 ++-- .../opensearch/benchmark-test.jenkinsfile.txt | 10 +++--- .../secure-benchmark-test.jenkinsfile.txt | 16 ++++----- 11 files changed, 90 insertions(+), 41 deletions(-) create mode 100644 tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt rename tests/jenkins/jenkinsjob-regression-files/opensearch/{benchmark-test-endpoint.jenkinsfile.txt => benchmark-test-endpoint-secure.jenkinsfile.txt} (93%) diff --git a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile index 77f3042e4e..f676d6fa3a 100644 --- a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile +++ b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@6.4.1', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', @@ -30,7 +30,7 @@ parameters { trim: true ) booleanParam( - name: 'CLUSTER_WITH_SECURITY', + name: 'SECURITY_ENABLED', description: 'Mention if the cluster is secured or insecured.', defaultValue: false, ) @@ -41,11 +41,6 @@ parameters { defaultValue: 'nyc_taxis', trim: true ) - string( - name: 'ADDITIONAL_CONFIG', - description: 'Additional opensearch.yml config parameters passed as JSON. e.g., `opensearch.experimental.feature.segment_replication_experimental.enabled:true cluster.indices.replication.strategy:SEGMENT`', - trim: true - ) string( name: 'USER_TAGS', description: 'Attach arbitrary text to the meta-data of each benchmark metric record, without any spaces. e.g., `run-type:adhoc,segrep:enabled,arch:x64`. ', @@ -110,18 +105,17 @@ parameters { steps { script { echo "CLUSTER_ENDPOINT: ${CLUSTER_ENDPOINT}" - echo "security-enabled: ${CLUSTER_WITH_SECURITY}" + echo "security-enabled: ${SECURITY_ENABLED}" runBenchmarkTestScript( endpoint: CLUSTER_ENDPOINT, - insecure: !CLUSTER_WITH_SECURITY, + insecure: !SECURITY_ENABLED, workload: TEST_WORKLOAD, - userTag: USER_TAGS.isEmpty() ? "security-enabled:${CLUSTER_WITH_SECURITY}" : "${USER_TAGS},security-enabled:${CLUSTER_WITH_SECURITY}", + userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}", workloadParams: WORKLOAD_PARAMS, testProcedure: TEST_PROCEDURE, excludeTasks: EXCLUDE_TASKS, includeTasks: INCLUDE_TASKS, - additionalConfig: ADDITIONAL_CONFIG, captureNodeStat: CAPTURE_NODE_STAT, captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT, telemetryParams: TELEMETRY_PARAMS diff --git a/jenkins/opensearch/benchmark-test.jenkinsfile b/jenkins/opensearch/benchmark-test.jenkinsfile index af2d4e0e32..f318c3ab7e 100644 --- a/jenkins/opensearch/benchmark-test.jenkinsfile +++ b/jenkins/opensearch/benchmark-test.jenkinsfile @@ -7,7 +7,7 @@ * compatible open source license. */ -lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([ +lib = library(identifier: 'jenkins@6.4.1', retriever: modernSCM([ $class: 'GitSCMSource', remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git', diff --git a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy index 4a1a463524..dc30ce0db7 100644 --- a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy +++ b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy @@ -26,7 +26,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -51,7 +51,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') binding.setVariable('CLUSTER_ENDPOINT', 'opensearch-ABCxdfdfhyfk.com') - binding.setVariable('CLUSTER_WITH_SECURITY', true) + binding.setVariable('SECURITY_ENABLED', true) binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') binding.setVariable('GITHUB_USER', 'test_user') binding.setVariable('GITHUB_TOKEN', 'test_token') @@ -73,7 +73,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ @Test public void testRunSecureBenchmarkTestScript_verifyPipeline() { super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", - "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile") + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile") } @Test @@ -108,6 +108,27 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ "./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) } + @Test + public void testRunSecureBenchmarkTestWithoutSecurity_verifyPipeline() { + binding.setVariable('SECURITY_ENABLED', false) + super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", + "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile") + } + + @Test + void testRunSecureBenchmarkTestScript_verifyWithoutSecurity() { + binding.setVariable('SECURITY_ENABLED', false) + runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItems( + "./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --test-procedure append-no-conflicts --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + )) + } def getCommandExecutions(methodName, command) { def shCommands = helper.callStack.findAll { diff --git a/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy b/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy index 1081bdbf35..c857cda436 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy @@ -26,7 +26,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest { void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -131,7 +131,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItems( - "./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100 ".toString() + "./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100".toString() )) } diff --git a/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy b/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy index 38be1b7d39..8d63f62e9a 100644 --- a/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy @@ -26,7 +26,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -140,7 +140,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(1)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100".toString() )) } diff --git a/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy b/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy index 417668b176..388ee400aa 100644 --- a/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunSecureBenchmarkTestScript.groovy @@ -26,7 +26,7 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ void setUp() { helper.registerSharedLibrary( library().name('jenkins') - .defaultVersion('5.11.0') + .defaultVersion('6.4.1') .allowOverride(true) .implicit(true) .targetPath('vars') @@ -141,8 +141,8 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{ assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItems( - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(), - "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(), + "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) } diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt index 15705c09e7..712f51a6cd 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.11.0, retriever=null}) + benchmark-test.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) benchmark-test.logRotator({daysToKeepStr=30}) @@ -133,7 +133,7 @@ benchmark-test.echo(DISTRIBUTION_VERSION: 3.0.0) benchmark-test.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://artifacts.com/artifact.tar.gz, distributionVersion=3.0.0, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, userTag=run-type:test,security-enabled:false,jenkins-build-id:307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) @@ -145,7 +145,7 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt new file mode 100644 index 0000000000..05ce8e79b5 --- /dev/null +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt @@ -0,0 +1,34 @@ + benchmark-test-endpoint.run() + benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) + benchmark-test-endpoint.library({identifier=jenkins@6.4.1, retriever=null}) + benchmark-test-endpoint.pipeline(groovy.lang.Closure) + benchmark-test-endpoint.timeout({time=24, unit=HOURS}) + benchmark-test-endpoint.logRotator({daysToKeepStr=30}) + benchmark-test-endpoint.buildDiscarder(null) + benchmark-test-endpoint.echo(Executing on agent [label:none]) + benchmark-test-endpoint.stage(validate-and-set-parameters, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) + benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure) + benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) + benchmark-test-endpoint.script(groovy.lang.Closure) + benchmark-test-endpoint.echo(CLUSTER_ENDPOINT: opensearch-ABCxdfdfhyfk.com) + benchmark-test-endpoint.echo(security-enabled: false) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=true, workload=nyc-taxis, userTag=run-type:test,security-enabled:false, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + benchmark-test-endpoint.postCleanup() + postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt similarity index 93% rename from tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt rename to tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt index b25e2401ad..b215d20f85 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test-endpoint.run() benchmark-test-endpoint.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test-endpoint.library({identifier=jenkins@cluster, retriever=null}) + benchmark-test-endpoint.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test-endpoint.pipeline(groovy.lang.Closure) benchmark-test-endpoint.timeout({time=24, unit=HOURS}) benchmark-test-endpoint.logRotator({daysToKeepStr=30}) @@ -16,9 +16,9 @@ benchmark-test-endpoint.script(groovy.lang.Closure) benchmark-test-endpoint.echo(CLUSTER_ENDPOINT: opensearch-ABCxdfdfhyfk.com) benchmark-test-endpoint.echo(security-enabled: true) - benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, additionalConfig=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt index 91b733e482..885bbfd4b2 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.11.0, retriever=null}) + benchmark-test.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) benchmark-test.logRotator({daysToKeepStr=30}) @@ -120,7 +120,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() @@ -139,7 +139,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() @@ -148,7 +148,7 @@ benchmark-test.echo(Architecture: x64) benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5-4xlarge, userTag=run-type:test,security-enabled:false,jenkins-build-id:307, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=type:search,scroll, includeTasks=type:search,scroll, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -165,7 +165,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100) Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt index 5736c7771b..247d987f53 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/secure-benchmark-test.jenkinsfile.txt @@ -1,6 +1,6 @@ benchmark-test.run() benchmark-test.modernSCM({$class=GitSCMSource, remote=https://github.com/opensearch-project/opensearch-build-libraries.git}) - benchmark-test.library({identifier=jenkins@5.11.0, retriever=null}) + benchmark-test.library({identifier=jenkins@6.4.1, retriever=null}) benchmark-test.pipeline(groovy.lang.Closure) benchmark-test.timeout({time=24, unit=HOURS}) benchmark-test.logRotator({daysToKeepStr=30}) @@ -120,7 +120,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -138,7 +138,7 @@ benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -147,7 +147,7 @@ benchmark-test.echo(Architecture: x64) benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc-taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=run-type:test,security-enabled:true,jenkins-build-id:307, workloadParams=, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -164,7 +164,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307 --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) @@ -184,7 +184,7 @@ Benchmark tests with security for 1234 completed}) benchmark-test.script(groovy.lang.Closure) benchmark-test.downloadBuildManifest({url=test://artifact.url, path=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) downloadBuildManifest.legacySCM(groovy.lang.Closure) - downloadBuildManifest.library({identifier=jenkins@5.11.0, retriever=null}) + downloadBuildManifest.library({identifier=jenkins@6.4.1, retriever=null}) downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/jenkins/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() @@ -193,7 +193,7 @@ Benchmark tests with security for 1234 completed}) benchmark-test.echo(Architecture: x64) benchmark-test.runBenchmarkTestScript({bundleManifest=tests/jenkins/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc-taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=run-type:test,security-enabled:false,jenkins-build-id:307, workloadParams=, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=, dataStorageSize=100, mlStorageSize=, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) - runBenchmarkTestScript.library({identifier=jenkins@5.11.0, retriever=null}) + runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/jenkins/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -210,7 +210,7 @@ Benchmark tests with security for 1234 completed}) BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{"telemetry_setting":"value"}') Messages.asBoolean() Messages.asBoolean() Messages.get([benchmark-test]) From dc9a030f4ff1f36b4e46081963189a9b3163e4c8 Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Fri, 5 Apr 2024 09:32:50 -0700 Subject: [PATCH 3/4] Add parameter change Signed-off-by: Divya Madala --- jenkins/opensearch/benchmark-test-endpoint.jenkinsfile | 3 +-- tests/jenkins/TestRunBenchmarkTestEndpoint.groovy | 7 ++++--- .../benchmark-test-endpoint-insecure.jenkinsfile.txt | 7 +++---- .../benchmark-test-endpoint-secure.jenkinsfile.txt | 1 - 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile index f676d6fa3a..3fe832ee23 100644 --- a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile +++ b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile @@ -104,12 +104,11 @@ parameters { agent { label AGENT_LABEL } steps { script { - echo "CLUSTER_ENDPOINT: ${CLUSTER_ENDPOINT}" echo "security-enabled: ${SECURITY_ENABLED}" runBenchmarkTestScript( endpoint: CLUSTER_ENDPOINT, - insecure: !SECURITY_ENABLED, + insecure: !(params.SECURITY_ENABLED), workload: TEST_WORKLOAD, userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}", workloadParams: WORKLOAD_PARAMS, diff --git a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy index dc30ce0db7..8ed48c0ef3 100644 --- a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy +++ b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy @@ -51,7 +51,6 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') binding.setVariable('CLUSTER_ENDPOINT', 'opensearch-ABCxdfdfhyfk.com') - binding.setVariable('SECURITY_ENABLED', true) binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') binding.setVariable('GITHUB_USER', 'test_user') binding.setVariable('GITHUB_TOKEN', 'test_token') @@ -72,6 +71,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ @Test public void testRunSecureBenchmarkTestScript_verifyPipeline() { + addParam('SECURITY_ENABLED', true) super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile") } @@ -97,6 +97,7 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ @Test void testRunSecureBenchmarkTestScript_verifyScriptExecutions() { + addParam('SECURITY_ENABLED', true) runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { @@ -110,14 +111,14 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ } @Test public void testRunSecureBenchmarkTestWithoutSecurity_verifyPipeline() { - binding.setVariable('SECURITY_ENABLED', false) + addParam('SECURITY_ENABLED', true) super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile") } @Test void testRunSecureBenchmarkTestScript_verifyWithoutSecurity() { - binding.setVariable('SECURITY_ENABLED', false) + addParam('SECURITY_ENABLED', false) runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile") def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt index 05ce8e79b5..8a4641830a 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile.txt @@ -14,9 +14,8 @@ benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure) benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) benchmark-test-endpoint.script(groovy.lang.Closure) - benchmark-test-endpoint.echo(CLUSTER_ENDPOINT: opensearch-ABCxdfdfhyfk.com) - benchmark-test-endpoint.echo(security-enabled: false) - benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=true, workload=nyc-taxis, userTag=run-type:test,security-enabled:false, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + benchmark-test-endpoint.echo(security-enabled: true) + benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@6.4.1, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -29,6 +28,6 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true --test-procedure append-no-conflicts --telemetry-params '{"telemetry_setting":"value"}') benchmark-test-endpoint.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true}) diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt index b215d20f85..8a4641830a 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile.txt @@ -14,7 +14,6 @@ benchmark-test-endpoint.stage(benchmark-test-with-cluster, groovy.lang.Closure) benchmark-test-endpoint.echo(Executing on agent [label:Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host]) benchmark-test-endpoint.script(groovy.lang.Closure) - benchmark-test-endpoint.echo(CLUSTER_ENDPOINT: opensearch-ABCxdfdfhyfk.com) benchmark-test-endpoint.echo(security-enabled: true) benchmark-test-endpoint.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc-taxis, userTag=run-type:test,security-enabled:true, workloadParams=, testProcedure=append-no-conflicts, excludeTasks=, includeTasks=, captureNodeStat=false, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) From 9d8f4c43c4f957973d5c07213973df34fc2619ca Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Mon, 8 Apr 2024 10:19:14 -0700 Subject: [PATCH 4/4] Remove JOB NAME Signed-off-by: Divya Madala --- jenkins/opensearch/benchmark-test-endpoint.jenkinsfile | 1 - tests/jenkins/TestRunBenchmarkTestEndpoint.groovy | 1 - 2 files changed, 2 deletions(-) diff --git a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile index 3fe832ee23..ed67151de0 100644 --- a/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile +++ b/jenkins/opensearch/benchmark-test-endpoint.jenkinsfile @@ -21,7 +21,6 @@ pipeline { } environment { AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test' - JOB_NAME = 'benchmark-test' } parameters { string( diff --git a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy index 8ed48c0ef3..34535ad41c 100644 --- a/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy +++ b/tests/jenkins/TestRunBenchmarkTestEndpoint.groovy @@ -60,7 +60,6 @@ class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{ binding.setVariable('EXCLUDE_TASKS', '') binding.setVariable('INCLUDE_TASKS', '') binding.setVariable('ADDITIONAL_CONFIG', '') - binding.setVariable('JOB_NAME', 'benchmark-test') binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') binding.setVariable('STAGE_NAME', 'test_stage') binding.setVariable('TEST_WORKLOAD', 'nyc-taxis')