From 2712ae4f629d55c4be9c77ca8df058654d708f03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 28 Mar 2024 21:47:22 +0000 Subject: [PATCH] Add changes to benchmark groovy library to support test runs against any endpoint (#400) Signed-off-by: Divya Madala (cherry picked from commit 71c80799ac2525804e6808f56ccd0624945398f4) Signed-off-by: github-actions[bot] --- build.gradle | 2 +- .../jenkins/TestRunBenchmarkTestScript.groovy | 4 +- ...TestRunBenchmarkTestScriptMultiNode.groovy | 4 +- .../TestRunBenchmarkTestScriptNoTags.groovy | 4 +- ...tRunBenchmarkTestScriptWithEndpoint.groovy | 73 +++++++++++++ .../BenchmarkTestMultinode_Jenkinsfile.txt | 4 +- .../BenchmarkTestNoManifest_Jenkinsfile.txt | 4 +- .../BenchmarkTestWithEndpoint_Jenkinsfile | 33 ++++++ .../BenchmarkTestWithEndpoint_Jenkinsfile.txt | 19 ++++ .../jobs/BenchmarkTest_Jenkinsfile.txt | 4 +- .../RunBenchmarkTestEndpointLibTest.groovy | 102 ++++++++++++++++++ vars/runBenchmarkTestScript.groovy | 63 +++++------ 12 files changed, 273 insertions(+), 43 deletions(-) create mode 100644 tests/jenkins/TestRunBenchmarkTestScriptWithEndpoint.groovy create mode 100644 tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile create mode 100644 tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile.txt create mode 100644 tests/jenkins/lib-testers/RunBenchmarkTestEndpointLibTest.groovy diff --git a/build.gradle b/build.gradle index 3a75f3936..77c625b05 100644 --- a/build.gradle +++ b/build.gradle @@ -121,7 +121,7 @@ jacocoTestReport { } } -String version = '6.4.0' +String version = '6.4.1' task updateVersion { doLast { diff --git a/tests/jenkins/TestRunBenchmarkTestScript.groovy b/tests/jenkins/TestRunBenchmarkTestScript.groovy index 5f6382ce4..7cada503b 100644 --- a/tests/jenkins/TestRunBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScript.groovy @@ -87,10 +87,10 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200" )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200" )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy index 421aa17a9..00fad8930 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy @@ -62,10 +62,10 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy index dca9018d9..818214a49 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy @@ -61,10 +61,10 @@ class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200" )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200" )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptWithEndpoint.groovy b/tests/jenkins/TestRunBenchmarkTestScriptWithEndpoint.groovy new file mode 100644 index 000000000..722982102 --- /dev/null +++ b/tests/jenkins/TestRunBenchmarkTestScriptWithEndpoint.groovy @@ -0,0 +1,73 @@ + +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +import jenkins.tests.BuildPipelineTest +import org.junit.Before +import org.junit.Test + +import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString +import static org.hamcrest.CoreMatchers.equalTo +import static org.hamcrest.CoreMatchers.hasItem +import static org.hamcrest.MatcherAssert.assertThat + +class TestRunBenchmarkTestScriptWithEndpoint extends BuildPipelineTest { + + @Before + void setUp() { + this.registerLibTester(new RunBenchmarkTestEndpointLibTester( + 'opensearch-ABCxdfdfhyfk.com', + 'false', + 'nyc_taxis', + 'true', + '', + '', + '', + '', + '', + '' + )) + super.setUp() + } + + @Test + public void testRunBenchmarkTestScript_PipelineSingleNode() { + super.testPipeline("tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile") + } + + @Test + void testRunBenchmarkTestScript_verifyScriptExecutionsNoManifest() { + runScript("tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile") + + def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll { + shCommand -> shCommand.contains('./test.sh') + } + + assertThat(testScriptCommands.size(), equalTo(1)) + assertThat(testScriptCommands, hasItem( + "./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag true,security-enabled:true" + )) + } + + def getCommandExecutions(methodName, command) { + def shCommands = helper.callStack.findAll { + call -> + call.methodName == methodName + }. + collect { + call -> + callArgsToString(call) + }.findAll { + shCommand -> + shCommand.contains(command) + } + + return shCommands + } +} diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt index d3f3c4f7a..acbd17ad1 100644 --- a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt @@ -28,7 +28,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') BenchmarkTestMultinode_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTestMultinode_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) @@ -56,4 +56,4 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') diff --git a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt index 968c3e76c..0c3ef7224 100644 --- a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt @@ -17,7 +17,7 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200) BenchmarkTestNoManifest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) @@ -34,4 +34,4 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200) diff --git a/tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile new file mode 100644 index 000000000..4bf5eed08 --- /dev/null +++ b/tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile @@ -0,0 +1,33 @@ + /* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +pipeline { + agent none + stages { + stage('benchmark-test') { + stage('test-with-security') { + steps { + script { + runBenchmarkTestScript( + endpoint: CLUSTER_ENDPOINT, + insecure: "false", + workload: TEST_WORKLOAD, + userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true", + workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, + additionalConfig: ADDITIONAL_CONFIG, + telemetryParams: TELEMETRY_PARAMS + ) + } + } + } + } + } +} \ No newline at end of file diff --git a/tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile.txt new file mode 100644 index 000000000..27abd5dd8 --- /dev/null +++ b/tests/jenkins/jobs/BenchmarkTestWithEndpoint_Jenkinsfile.txt @@ -0,0 +1,19 @@ + BenchmarkTestWithEndpoint_Jenkinsfile.run() + BenchmarkTestWithEndpoint_Jenkinsfile.pipeline(groovy.lang.Closure) + BenchmarkTestWithEndpoint_Jenkinsfile.echo(Executing on agent [label:none]) + BenchmarkTestWithEndpoint_Jenkinsfile.stage(test-with-security, groovy.lang.Closure) + BenchmarkTestWithEndpoint_Jenkinsfile.script(groovy.lang.Closure) + BenchmarkTestWithEndpoint_Jenkinsfile.runBenchmarkTestScript({endpoint=opensearch-ABCxdfdfhyfk.com, insecure=false, workload=nyc_taxis, userTag=true,security-enabled:true, workloadParams=, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=, telemetryParams=}) + runBenchmarkTestScript.legacySCM(groovy.lang.Closure) + runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) + runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) + runBenchmarkTestScript.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME}) + runBenchmarkTestScript.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME], groovy.lang.Closure) + runBenchmarkTestScript.withAWS({role=opensearch-test, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + runBenchmarkTestScript.s3Download({file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-user, variable=DATASTORE_USER}) + runBenchmarkTestScript.string({credentialsId=benchmark-metrics-datastore-password, variable=DATASTORE_PASSWORD}) + runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) + runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) + runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) + runBenchmarkTestScript.sh(./test.sh benchmark-test --cluster-endpoint opensearch-ABCxdfdfhyfk.com --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag true,security-enabled:true) diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt index 297323aae..0be218926 100644 --- a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt @@ -28,7 +28,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200) BenchmarkTest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTest_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) @@ -56,4 +56,4 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200) diff --git a/tests/jenkins/lib-testers/RunBenchmarkTestEndpointLibTest.groovy b/tests/jenkins/lib-testers/RunBenchmarkTestEndpointLibTest.groovy new file mode 100644 index 000000000..5bb231ed3 --- /dev/null +++ b/tests/jenkins/lib-testers/RunBenchmarkTestEndpointLibTest.groovy @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +import static org.hamcrest.CoreMatchers.notNullValue +import static org.hamcrest.MatcherAssert.assertThat + +class RunBenchmarkTestEndpointLibTester extends LibFunctionTester{ + + private String endpoint + private String insecure + private String workload + private String userTag + private String workloadParams + private String testProcedure + private String excludeTasks + private String includeTasks + private String additionalConfig + private String telemetryParams + + public RunBenchmarkTestEndpointLibTester(endpoint, insecure, workload, userTag, workloadParams, + testProcedure, excludeTasks, includeTasks, + additionalConfig,telemetryParams){ + this.endpoint = endpoint + this.insecure = insecure + this.workload = workload + this.userTag = userTag + this.workloadParams = workloadParams + this.testProcedure = testProcedure + this.excludeTasks = excludeTasks + this.includeTasks = includeTasks + this.additionalConfig = additionalConfig + this.telemetryParams = telemetryParams + } + + + @Override + String libFunctionName() { + return 'runBenchmarkTestScript' + } + + @Override + void parameterInvariantsAssertions(Object call) { + if (!this.insecure.isEmpty()) { + assertThat(call.args.insecure.first(), notNullValue()) + } + if (!this.workload.isEmpty()) { + assertThat(call.args.workload.first(), notNullValue()) + } + } + + @Override + boolean expectedParametersMatcher(Object call) { + return call.args.endpoint.first().toString().equals(this.endpoint) + } + + @Override + void configure(Object helper, Object binding) { + helper.registerAllowedMethod("s3Download", [Map]) + helper.registerAllowedMethod("uploadTestResults", [Map]) + helper.registerAllowedMethod("s3Upload", [Map]) + helper.registerAllowedMethod("withAWS", [Map, Closure], { + args, + closure -> + closure.delegate = delegate + return helper.callClosure(closure) + }) + helper.registerAllowedMethod('findFiles', [Map.class], null) + helper.registerAllowedMethod("withCredentials", [Map]) + helper.registerAllowedMethod('parameterizedCron', [String], null) + binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host') + binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1') + binding.setVariable('ARCHITECTURE', 'x64') + binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket') + binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role') + binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account') + binding.setVariable('env', ['BUILD_NUMBER': '307']) + binding.setVariable('BUILD_NUMBER', '307') + binding.setVariable('BUILD_URL', 'test://artifact.url') + binding.setVariable('CLUSTER_ENDPOINT', endpoint) + binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name') + binding.setVariable('GITHUB_USER', 'test_user') + binding.setVariable('GITHUB_TOKEN', 'test_token') + binding.setVariable('USER_TAGS', userTag) + binding.setVariable('WORKLOAD_PARAMS', workloadParams) + binding.setVariable('TEST_PROCEDURE', testProcedure) + binding.setVariable('EXCLUDE_TASKS', excludeTasks) + binding.setVariable('INCLUDE_TASKS', includeTasks) + binding.setVariable('ADDITIONAL_CONFIG', additionalConfig) + binding.setVariable('JOB_NAME', 'benchmark-test') + binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config') + binding.setVariable('PUBLIC_ARTIFACT_URL', 'test://artifact.url') + binding.setVariable('STAGE_NAME', 'test_stage') + binding.setVariable('TEST_WORKLOAD', workload) + binding.setVariable('WEBHOOK_URL', 'test://artifact.url') + binding.setVariable('TELEMETRY_PARAMS', telemetryParams) + } +} diff --git a/vars/runBenchmarkTestScript.groovy b/vars/runBenchmarkTestScript.groovy index 2a38c63fd..427bc1d74 100644 --- a/vars/runBenchmarkTestScript.groovy +++ b/vars/runBenchmarkTestScript.groovy @@ -12,6 +12,7 @@ * @param args.bundleManifest - OpenSearch bundle manifest url. * @param args.distributionUrl - Download link for the OpenSearch bundle tarball. * @param args.distributionVersion - Provide OpenSearch version if using distributionUrl param + * @param args.endpoint - Endpoint to the cluster. * @param args.insecure - Force the security of the cluster to be disabled, default is false. * @param args.workload - Name of the workload that OpenSearch Benchmark should run, default is nyc_taxis. * @param args.singleNode - Create single node OS cluster, default is true. @@ -51,7 +52,9 @@ void call(Map args = [:]) { withCredentials([string(credentialsId: 'jenkins-aws-account-public', variable: 'AWS_ACCOUNT_PUBLIC'), string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) { withAWS(role: 'opensearch-test', roleAccount: "${AWS_ACCOUNT_PUBLIC}", duration: 900, roleSessionName: 'jenkins-session') { - s3Download(file: 'config.yml', bucket: "${ARTIFACT_BUCKET_NAME}", path: "${BENCHMARK_TEST_CONFIG_LOCATION}/${config_name}", force: true) + if(isNullOrEmpty(args.endpoint)) { + s3Download(file: 'config.yml', bucket: "${ARTIFACT_BUCKET_NAME}", path: "${BENCHMARK_TEST_CONFIG_LOCATION}/${config_name}", force: true) + } s3Download(file: 'benchmark.ini', bucket: "${ARTIFACT_BUCKET_NAME}", path: "${BENCHMARK_TEST_CONFIG_LOCATION}/${benchmark_config}", force: true) /*Added sleep to let the file get downloaded first before write happens. Without the sleep the write is @@ -71,38 +74,38 @@ void call(Map args = [:]) { sh([ './test.sh', 'benchmark-test', - isNullOrEmpty(args.bundleManifest.toString()) ? "" : "--bundle-manifest ${args.bundleManifest}", - isNullOrEmpty(args.distributionUrl.toString()) ? "" : "--distribution-url ${args.distributionUrl}", - isNullOrEmpty(args.distributionVersion.toString()) ? "" : "--distribution-version ${args.distributionVersion}", - "--config ${WORKSPACE}/config.yml", + isNullOrEmpty(args.bundleManifest) ? "" : "--bundle-manifest ${args.bundleManifest}", + isNullOrEmpty(args.distributionUrl) ? "" : "--distribution-url ${args.distributionUrl}", + isNullOrEmpty(args.distributionVersion) ? "" : "--distribution-version ${args.distributionVersion}", + isNullOrEmpty(args.endpoint) ? "" : "--cluster-endpoint ${args.endpoint}", + isNullOrEmpty(args.endpoint) ? "--config ${WORKSPACE}/config.yml" : "", "--workload ${args.workload}", "--benchmark-config ${WORKSPACE}/benchmark.ini", "--user-tag ${userTags}", - args.insecure.toBoolean() ? "--without-security" : "", - args.singleNode.toBoolean() ? "--single-node" : "", - args.minDistribution.toBoolean() ? "--min-distribution" : "", - args.use50PercentHeap.toBoolean() ? "--use-50-percent-heap" : "", - args.enableRemoteStore.toBoolean() ? "--enable-remote-store" : "", - args.captureNodeStat.toBoolean() ? "--capture-node-stat" : "", - args.captureSegmentReplicationStat.toBoolean() ? "--capture-segment-replication-stat" : "", - isNullOrEmpty(args.suffix.toString()) ? "" : "--suffix ${args.suffix}", - isNullOrEmpty(args.managerNodeCount.toString()) ? "" : "--manager-node-count ${args.managerNodeCount}", - isNullOrEmpty(args.dataNodeCount.toString()) ? "" : "--data-node-count ${args.dataNodeCount}", - isNullOrEmpty(args.clientNodeCount.toString()) ? "" : "--client-node-count ${args.clientNodeCount}", - isNullOrEmpty(args.ingestNodeCount.toString()) ? "" : "--ingest-node-count ${args.ingestNodeCount}", - isNullOrEmpty(args.mlNodeCount.toString()) ? "" : "--ml-node-count ${args.mlNodeCount}", - isNullOrEmpty(args.dataInstanceType.toString()) ? "" : "--data-instance-type ${args.dataInstanceType}", - isNullOrEmpty(args.workloadParams.toString()) ? "" : "--workload-params '${args.workloadParams}'", - isNullOrEmpty(args.testProcedure.toString()) ? "" : "--test-procedure ${args.testProcedure}", - isNullOrEmpty(args.excludeTasks.toString()) ? "" : "--exclude-tasks ${args.excludeTasks}", - isNullOrEmpty(args.includeTasks.toString()) ? "" : "--include-tasks ${args.includeTasks}", - isNullOrEmpty(args.additionalConfig.toString()) ? "" : "--additional-config ${args.additionalConfig}", - isNullOrEmpty(args.dataStorageSize.toString()) ? "" : "--data-node-storage ${args.dataStorageSize}", - isNullOrEmpty(args.mlStorageSize.toString()) ? "" : "--ml-node-storage ${args.mlStorageSize}", - isNullOrEmpty(args.jvmSysProps.toString()) ? "" : "--jvm-sys-props ${args.jvmSysProps}", - isNullOrEmpty(args.telemetryParams.toString()) ? "" : "--telemetry-params '${args.telemetryParams}'" - ].join(' ')) - + args.insecure?.toBoolean() ? "--without-security" : "", + args.singleNode?.toBoolean() ? "--single-node" : "", + args.minDistribution?.toBoolean() ? "--min-distribution" : "", + args.use50PercentHeap?.toBoolean() ? "--use-50-percent-heap" : "", + args.enableRemoteStore?.toBoolean() ? "--enable-remote-store" : "", + args.captureNodeStat?.toBoolean() ? "--capture-node-stat" : "", + args.captureSegmentReplicationStat?.toBoolean() ? "--capture-segment-replication-stat" : "", + isNullOrEmpty(args.suffix) ? "" : "--suffix ${args.suffix}", + isNullOrEmpty(args.managerNodeCount) ? "" : "--manager-node-count ${args.managerNodeCount}", + isNullOrEmpty(args.dataNodeCount) ? "" : "--data-node-count ${args.dataNodeCount}", + isNullOrEmpty(args.clientNodeCount) ? "" : "--client-node-count ${args.clientNodeCount}", + isNullOrEmpty(args.ingestNodeCount) ? "" : "--ingest-node-count ${args.ingestNodeCount}", + isNullOrEmpty(args.mlNodeCount) ? "" : "--ml-node-count ${args.mlNodeCount}", + isNullOrEmpty(args.dataInstanceType) ? "" : "--data-instance-type ${args.dataInstanceType}", + isNullOrEmpty(args.workloadParams) ? "" : "--workload-params '${args.workloadParams}'", + isNullOrEmpty(args.testProcedure) ? "" : "--test-procedure ${args.testProcedure}", + isNullOrEmpty(args.excludeTasks) ? "" : "--exclude-tasks ${args.excludeTasks}", + isNullOrEmpty(args.includeTasks) ? "" : "--include-tasks ${args.includeTasks}", + isNullOrEmpty(args.additionalConfig) ? "" : "--additional-config ${args.additionalConfig}", + isNullOrEmpty(args.dataStorageSize) ? "" : "--data-node-storage ${args.dataStorageSize}", + isNullOrEmpty(args.mlStorageSize) ? "" : "--ml-node-storage ${args.mlStorageSize}", + isNullOrEmpty(args.jvmSysProps) ? "" : "--jvm-sys-props ${args.jvmSysProps}", + isNullOrEmpty(args.telemetryParams) ? "" : "--telemetry-params '${args.telemetryParams}'" + ].join(' ').trim()) } void editBenchmarkConfig(String config_file) {