Add jenkins benchmark job for cluster endpoint runs

jenkins/opensearch/benchmark-test-endpoint.jenkinsfile

@@ -0,0 +1,130 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+lib = library(identifier: '[email protected]', retriever: modernSCM([
+
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
+]))
+
+pipeline {
+    agent none
+    options {
+        timeout(time: 24, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '30'))
+    }
+    environment {
+        AGENT_LABEL = 'Jenkins-Agent-AL2023-X64-M52xlarge-Benchmark-Test'
+    }
+parameters {
+        string(
+            name: 'CLUSTER_ENDPOINT',
+            description: 'Provide an endpoint to a cluster for running benchmark tests against it.',
+            trim: true
+        )
+        booleanParam(
+            name: 'SECURITY_ENABLED',
+            description: 'Mention if the cluster is secured or insecured.',
+            defaultValue: false,
+        )
+        string(
+
+            name: 'TEST_WORKLOAD',
+            description: 'The workload name from OpenSearch Benchmark Workloads.',
+            defaultValue: 'nyc_taxis',
+            trim: true
+        )
+        string(
+            name: 'USER_TAGS',
+            description: 'Attach arbitrary text to the meta-data of each benchmark metric record, without any spaces. e.g., `run-type:adhoc,segrep:enabled,arch:x64`. ',
+            trim: true
+        )
+        string(
+            name: 'WORKLOAD_PARAMS',
+            description: 'With this parameter you can inject variables into workloads. Use json type. e.g., `{"number_of_replicas":"1","number_of_shards":"5"}`',
+            trim: true
+        )
+        string(
+            name: 'TEST_PROCEDURE',
+            description: 'Defines a test procedure to use. e.g., `append-no-conflicts,significant-text`',
+            trim: true
+        )
+        string(
+            name: 'EXCLUDE_TASKS',
+            description: 'Defines a comma-separated list of test procedure tasks not to run. Default runs all. e.g., `type:search,delete-index`',
+            trim: true
+        )
+        string(
+            name: 'INCLUDE_TASKS',
+            description: 'Defines a comma-separated list of test procedure tasks to run. Default runs all. e.g., `type:search,delete-index`',
+            trim: true
+        )
+        booleanParam(
+            name: 'CAPTURE_NODE_STAT',
+            description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.',
+            defaultValue: false
+        )
+        booleanParam(
+            name: 'CAPTURE_SEGMENT_REPLICATION_STAT',
+            description: 'Enable opensearch-benchmark segment-replication-stats telemetry to capture metrics such as replication lag.',
+            defaultValue: false
+        )
+        string(
+            name: 'TELEMETRY_PARAMS',
+            description: 'Allows to set parameters for telemetry devices. Use json type. e.g.,{"node-stats-include-indices":"true","node-stats-include-indices-metrics":"segments"}',
+            trim: true
+        )
+    }
+
+    stages {
+        stage('validate-and-set-parameters') {
+            agent { label AGENT_LABEL }
+            steps {
+                script {
+                    if (CLUSTER_ENDPOINT == '') {
+                        currentBuild.result = 'ABORTED'
+                        error("Benchmark Tests failed to start. Provide CLUSTER_ENDPOINT to run tests")
+                    }
+                }
+            }
+            post {
+                always {
+                    postCleanup()
+                }
+            }
+        }
+        stage('benchmark-test-with-cluster') {
+            agent { label AGENT_LABEL }
+            steps {
+                script {
+                    echo "security-enabled: ${SECURITY_ENABLED}"
+
+                    runBenchmarkTestScript(
+                        endpoint: CLUSTER_ENDPOINT,
+                        insecure: !(params.SECURITY_ENABLED),
+                        workload: TEST_WORKLOAD,
+                        userTag: USER_TAGS.isEmpty() ? "security-enabled:${SECURITY_ENABLED}" : "${USER_TAGS},security-enabled:${SECURITY_ENABLED}",
+                        workloadParams: WORKLOAD_PARAMS,
+                        testProcedure: TEST_PROCEDURE,
+                        excludeTasks: EXCLUDE_TASKS,
+                        includeTasks: INCLUDE_TASKS,
+                        captureNodeStat: CAPTURE_NODE_STAT,
+                        captureSegmentReplicationStat: CAPTURE_SEGMENT_REPLICATION_STAT,
+                        telemetryParams: TELEMETRY_PARAMS
+                    )
+                }
+                post {
+                    always {
+                        postCleanup()
+                    }
+                }
+            }
+        }
+    }
+}

jenkins/opensearch/benchmark-test.jenkinsfile

@@ -7,7 +7,7 @@
  * compatible open source license.
  */
 
-lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([
+lib = library(identifier: 'jenkins@6.4.1', retriever: modernSCM([
 
     $class: 'GitSCMSource',
     remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',

tests/jenkins/TestRunBenchmarkTestEndpoint.groovy

@@ -0,0 +1,148 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+import jenkins.tests.BuildPipelineTest
+import org.junit.Before
+import org.junit.Test
+
+import static com.lesfurets.jenkins.unit.MethodCall.callArgsToString
+import static org.hamcrest.CoreMatchers.equalTo
+import static org.hamcrest.CoreMatchers.hasItem
+import static org.hamcrest.CoreMatchers.hasItems
+import static org.hamcrest.MatcherAssert.assertThat
+
+import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
+import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
+
+class TestRunBenchmarkTestEndpoint extends BuildPipelineTest{
+    @Override
+    @Before
+    void setUp() {
+        helper.registerSharedLibrary(
+                library().name('jenkins')
+                        .defaultVersion('6.4.1')
+                        .allowOverride(true)
+                        .implicit(true)
+                        .targetPath('vars')
+                        .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
+                        .build()
+        )
+        helper.registerAllowedMethod("s3Download", [Map])
+        helper.registerAllowedMethod("uploadTestResults", [Map])
+        helper.registerAllowedMethod("s3Upload", [Map])
+        helper.registerAllowedMethod("withAWS", [Map, Closure], {
+            args,
+            closure ->
+                closure.delegate = delegate
+                return helper.callClosure(closure)
+        })
+        helper.registerAllowedMethod('findFiles', [Map.class], null)
+        helper.registerAllowedMethod("withCredentials", [Map])
+        binding.setVariable('AGENT_LABEL', 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host')
+        binding.setVariable('AGENT_IMAGE', 'opensearchstaging/ci-runner:ci-runner-centos7-v1')
+        binding.setVariable('ARCHITECTURE', 'x64')
+        binding.setVariable('ARTIFACT_BUCKET_NAME', 'test_bucket')
+        binding.setVariable('ARTIFACT_DOWNLOAD_ROLE_NAME', 'Dummy_Download_Role')
+        binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account')
+        binding.setVariable('CLUSTER_ENDPOINT', 'opensearch-ABCxdfdfhyfk.com')
+        binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'bot_token_name')
+        binding.setVariable('GITHUB_USER', 'test_user')
+        binding.setVariable('GITHUB_TOKEN', 'test_token')
+        binding.setVariable('USER_TAGS', 'run-type:test')
+        binding.setVariable('WORKLOAD_PARAMS', '')
+        binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts')
+        binding.setVariable('EXCLUDE_TASKS', '')
+        binding.setVariable('INCLUDE_TASKS', '')
+        binding.setVariable('ADDITIONAL_CONFIG', '')
+        binding.setVariable('BENCHMARK_TEST_CONFIG_LOCATION', 'test_config')
+        binding.setVariable('STAGE_NAME', 'test_stage')
+        binding.setVariable('TEST_WORKLOAD', 'nyc-taxis')
+        binding.setVariable('TELEMETRY_PARAMS', '{"telemetry_setting":"value"}')
+
+        super.setUp()
+    }
+
+    @Test
+    public void testRunSecureBenchmarkTestScript_verifyPipeline() {
+        addParam('SECURITY_ENABLED', true)
+        super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile",
+                "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-secure.jenkinsfile")
+    }
+
+    @Test
+    void testRunSecureBenchmarkTestScript_verifyArtifactDownloads() {
+        runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile")
+
+        def curlCommands = getCommandExecutions('sh', 'curl').findAll {
+            shCommand -> shCommand.contains('curl')
+        }
+
+        def s3DownloadCommands = getCommandExecutions('s3Download', 'bucket').findAll {
+            shCommand -> shCommand.contains('bucket')
+        }
+
+        assertThat(s3DownloadCommands.size(), equalTo(1))
+        assertThat(s3DownloadCommands, hasItems(
+                "{file=benchmark.ini, bucket=ARTIFACT_BUCKET_NAME, path=test_config/benchmark.ini, force=true}".toString()
+        ))
+    }
+
+
+    @Test
+    void testRunSecureBenchmarkTestScript_verifyScriptExecutions() {
+        addParam('SECURITY_ENABLED', true)
+        runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile")
+
+        def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
+            shCommand -> shCommand.contains('./test.sh')
+        }
+
+        assertThat(testScriptCommands.size(), equalTo(1))
+        assertThat(testScriptCommands, hasItems(
+                "./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:true                --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+        ))
+    }
+    @Test
+    public void testRunSecureBenchmarkTestWithoutSecurity_verifyPipeline() {
+        addParam('SECURITY_ENABLED', true)
+        super.testPipeline("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile",
+                "tests/jenkins/jenkinsjob-regression-files/opensearch/benchmark-test-endpoint-insecure.jenkinsfile")
+    }
+
+    @Test
+    void testRunSecureBenchmarkTestScript_verifyWithoutSecurity() {
+        addParam('SECURITY_ENABLED', false)
+        runScript("jenkins/opensearch/benchmark-test-endpoint.jenkinsfile")
+
+        def testScriptCommands = getCommandExecutions('sh', './test.sh').findAll {
+            shCommand -> shCommand.contains('./test.sh')
+        }
+
+        assertThat(testScriptCommands.size(), equalTo(1))
+        assertThat(testScriptCommands, hasItems(
+                "./test.sh benchmark-test    --cluster-endpoint opensearch-ABCxdfdfhyfk.com  --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security               --test-procedure append-no-conflicts       --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+        ))
+    }
+
+    def getCommandExecutions(methodName, command) {
+        def shCommands = helper.callStack.findAll {
+            call ->
+                call.methodName == methodName
+        }.
+                collect {
+                    call ->
+                        callArgsToString(call)
+                }.findAll {
+            shCommand ->
+                shCommand.contains(command)
+        }
+
+        return shCommands
+    }
+}

tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy

@@ -26,7 +26,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest {
     void setUp() {
         helper.registerSharedLibrary(
                 library().name('jenkins')
-                        .defaultVersion('5.11.0')
+                        .defaultVersion('6.4.1')
                         .allowOverride(true)
                         .implicit(true)
                         .targetPath('vars')
@@ -131,7 +131,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest {
 
         assertThat(testScriptCommands.size(), equalTo(1))
         assertThat(testScriptCommands, hasItems(
-                "./test.sh benchmark-test  --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security   --use-50-percent-heap    --suffix 307 --manager-node-count 3 --data-node-count 3    --data-instance-type r5-4xlarge  --test-procedure append-no-conflicts    --data-node-storage 100   ".toString()
+                "./test.sh benchmark-test  --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0  --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false,jenkins-build-id:307 --without-security   --use-50-percent-heap    --suffix 307 --manager-node-count 3 --data-node-count 3    --data-instance-type r5-4xlarge  --test-procedure append-no-conflicts    --data-node-storage 100".toString()
         ))
     }
 

tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy

@@ -26,7 +26,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{
     void setUp() {
         helper.registerSharedLibrary(
                 library().name('jenkins')
-                        .defaultVersion('5.11.0')
+                        .defaultVersion('6.4.1')
                         .allowOverride(true)
                         .implicit(true)
                         .targetPath('vars')
@@ -140,7 +140,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{
 
         assertThat(testScriptCommands.size(), equalTo(1))
         assertThat(testScriptCommands, hasItem(
-                "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml   --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security   --use-50-percent-heap    --suffix 307 --manager-node-count 3 --data-node-count 3    --data-instance-type r5-4xlarge  --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll  --data-node-storage 100   ".toString()
+                "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml    --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security   --use-50-percent-heap    --suffix 307 --manager-node-count 3 --data-node-count 3    --data-instance-type r5-4xlarge  --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll  --data-node-storage 100".toString()
         ))
     }
 

tests/jenkins/TestRunSecureBenchmarkTestScript.groovy

@@ -26,7 +26,7 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{
     void setUp() {
         helper.registerSharedLibrary(
                 library().name('jenkins')
-                        .defaultVersion('5.11.0')
+                        .defaultVersion('6.4.1')
                         .allowOverride(true)
                         .implicit(true)
                         .targetPath('vars')
@@ -141,8 +141,8 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{
 
         assertThat(testScriptCommands.size(), equalTo(2))
         assertThat(testScriptCommands, hasItems(
-                "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307    --use-50-percent-heap --enable-remote-store  --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3          --data-node-storage 100   --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(),
-                "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security   --use-50-percent-heap --enable-remote-store  --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3          --data-node-storage 100   --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+                "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml    --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true,jenkins-build-id:307    --use-50-percent-heap --enable-remote-store  --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3          --data-node-storage 100   --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(),
+                "./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml    --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false,jenkins-build-id:307 --without-security   --use-50-percent-heap --enable-remote-store  --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3          --data-node-storage 100   --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
         ))
     }