From 41ae02a39780f992f43973e6762b8bd4fec15218 Mon Sep 17 00:00:00 2001 From: Divya Madala Date: Wed, 8 May 2024 14:32:01 -0700 Subject: [PATCH] Add more changes Signed-off-by: Divya Madala --- .../benchmark_test_runner_opensearch.py | 12 +- .../benchmark_test/benchmark_test_suite.py | 33 +- .../test_benchmark_test_runner_opensearch.py | 16 +- .../test_benchmark_test_suite.py | 288 +++++++++--------- 4 files changed, 166 insertions(+), 183 deletions(-) diff --git a/src/test_workflow/benchmark_test/benchmark_test_runner_opensearch.py b/src/test_workflow/benchmark_test/benchmark_test_runner_opensearch.py index 5a7df886a1..a68cb4fdc1 100644 --- a/src/test_workflow/benchmark_test/benchmark_test_runner_opensearch.py +++ b/src/test_workflow/benchmark_test/benchmark_test_runner_opensearch.py @@ -7,7 +7,6 @@ import logging import os -import subprocess from typing import Union import yaml @@ -43,10 +42,8 @@ def run_tests(self) -> None: cluster = BenchmarkTestCluster(self.args) cluster.start() benchmark_test_suite = BenchmarkTestSuite(cluster.endpoint_with_port, self.security, self.args, cluster.fetch_password()) - try: - retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2) - finally: - subprocess.check_call(f"docker rm docker-container-{self.args.stack_suffix}", cwd=os.getcwd(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2) + else: config = yaml.safe_load(self.args.config) @@ -56,7 +53,4 @@ def run_tests(self) -> None: with WorkingDirectory(current_workspace): with BenchmarkCreateCluster.create(self.args, self.test_manifest, config, current_workspace) as test_cluster: benchmark_test_suite = BenchmarkTestSuite(test_cluster.endpoint_with_port, self.security, self.args, test_cluster.fetch_password()) - try: - retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2) - finally: - subprocess.check_call(f"docker rm docker-container-{self.args.stack_suffix}", cwd=os.getcwd(), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + retry_call(benchmark_test_suite.execute, tries=3, delay=60, backoff=2) diff --git a/src/test_workflow/benchmark_test/benchmark_test_suite.py b/src/test_workflow/benchmark_test/benchmark_test_suite.py index 371083b9be..7b95d52e33 100644 --- a/src/test_workflow/benchmark_test/benchmark_test_suite.py +++ b/src/test_workflow/benchmark_test/benchmark_test_suite.py @@ -81,19 +81,26 @@ def __init__( def execute(self) -> None: log_info = f"Executing {self.command.replace(self.endpoint, len(self.endpoint) * '*').replace(self.args.username, len(self.args.username) * '*')}" logging.info(log_info.replace(self.password, len(self.password) * '*') if self.password else log_info) - subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True) + try: + subprocess.check_call(f"{self.command}", cwd=os.getcwd(), shell=True) + if self.args.cluster_endpoint: + self.convert() + finally: + self.cleanup() + + def convert(self) -> None: with TemporaryDirectory() as work_dir: subprocess.check_call(f"docker cp docker-container-{self.args.stack_suffix}:opensearch-benchmark/. {str(work_dir.path)}", cwd=os.getcwd(), shell=True) file_path = glob.glob(os.path.join(str(work_dir.path), "test_executions", "*", "test_execution.json")) - self.convert(file_path[0]) - - def convert(self, results: str) -> None: - with open(results) as file: - data = json.load(file) - formatted_data = pd.json_normalize(data["results"]["op_metrics"]) - formatted_data.to_csv(os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.csv"), index=False) - df = pd.read_csv(os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.csv")) - pd.set_option('display.width', int(2 * shutil.get_terminal_size().columns)) - pd.set_option('display.max_rows', None) - pd.set_option('display.max_columns', None) - logging.info(f"\n{df}") + with open(file_path[0]) as file: + data = json.load(file) + formatted_data = pd.json_normalize(data["results"]["op_metrics"]) + formatted_data.to_csv(os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.csv"), index=False) + df = pd.read_csv(os.path.join(os.getcwd(), f"test_execution_{self.args.stack_suffix}.csv")) + pd.set_option('display.width', int(2 * shutil.get_terminal_size().columns)) + pd.set_option('display.max_rows', None) + pd.set_option('display.max_columns', None) + logging.info(f"\n{df}") + + def cleanup(self) -> None: + subprocess.check_call(f"docker rm docker-container-{self.args.stack_suffix}", cwd=os.getcwd(), shell=True) diff --git a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_runner_opensearch.py b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_runner_opensearch.py index d2294e9c0b..8ebd2f4140 100644 --- a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_runner_opensearch.py +++ b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_runner_opensearch.py @@ -29,10 +29,8 @@ class TestBenchmarkTestRunnerOpenSearch(unittest.TestCase): @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.GitRepository") @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkCreateCluster.create") @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkTestSuite") - @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') - def test_run(self, mock_check_call: Mock, mock_suite: Mock, mock_cluster: Mock, mock_git: Mock, mock_temp_directory: Mock, + def test_run(self, mock_suite: Mock, mock_cluster: Mock, mock_git: Mock, mock_temp_directory: Mock, *mocks: Any) -> None: - mock_check_call.return_value = 0 mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() mock_cluster.return_value.__enter__.return_value = mock_cluster @@ -61,11 +59,9 @@ def test_run(self, mock_check_call: Mock, mock_suite: Mock, mock_cluster: Mock, @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.GitRepository") @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkCreateCluster.create") @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkTestSuite") - @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') - def test_run_with_dist_url_and_version(self, mock_check_call: Mock, mock_suite: Mock, mock_cluster: Mock, mock_git: Mock, + def test_run_with_dist_url_and_version(self, mock_suite: Mock, mock_cluster: Mock, mock_git: Mock, mock_temp_directory: Mock, *mocks: Any) -> None: - mock_check_call.return_value = 0 mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() mock_cluster.return_value.__enter__.return_value = mock_cluster @@ -82,9 +78,7 @@ def test_run_with_dist_url_and_version(self, mock_check_call: Mock, mock_suite: @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkTestCluster.start") @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkTestSuite") @patch('test_workflow.benchmark_test.benchmark_test_runner_opensearch.retry_call') - @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') - def test_run_with_cluster_endpoint(self, mock_check_call: Mock, mock_retry_call: Mock, mock_suite: Mock, mock_benchmark_test_cluster: Mock) -> None: - mock_check_call.return_value = 0 + def test_run_with_cluster_endpoint(self, mock_retry_call: Mock, mock_suite: Mock, mock_benchmark_test_cluster: Mock) -> None: args = MagicMock(cluster_endpoint=True) mock_cluster = MagicMock() @@ -98,12 +92,10 @@ def test_run_with_cluster_endpoint(self, mock_check_call: Mock, mock_retry_call: @patch('test_workflow.benchmark_test.benchmark_test_cluster.BenchmarkTestCluster.wait_for_processing') @patch("test_workflow.benchmark_test.benchmark_test_runner_opensearch.BenchmarkTestSuite") @patch('test_workflow.benchmark_test.benchmark_test_runner_opensearch.retry_call') - @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch("subprocess.run") @patch("requests.get") - def test_run_with_cluster_endpoint_with_arguments(self, mock_requests_get: Mock, mock_subprocess_run: Mock, mock_check_call: Mock, + def test_run_with_cluster_endpoint_with_arguments(self, mock_requests_get: Mock, mock_subprocess_run: Mock, mock_retry_call: Mock, mock_suite: Mock, mock_wait_for_processing: Optional[Mock]) -> None: - mock_check_call.return_value = 0 args = MagicMock(cluster_endpoint=True) mock_wait_for_processing.return_value = None mock_result = MagicMock() diff --git a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py index 0fd67c8159..6f6c7bbfd7 100644 --- a/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py +++ b/tests/tests_test_workflow/test_benchmark_workflow/benchmark_test/test_benchmark_test_suite.py @@ -17,7 +17,6 @@ class TestBenchmarkTestSuite(unittest.TestCase): def setUp(self, **kwargs: Any) -> None: - self.maxDiff = None with patch('test_workflow.integ_test.utils.get_password') as mock_get_password: self.args = Mock() self.args.insecure = True @@ -27,6 +26,7 @@ def setUp(self, **kwargs: Any) -> None: mock_get_password.return_value = get_password('2.11.0') self.args.username = "admin" self.password = "myStrongPassword123!" + self.args.cluster_endpoint = None self.args.user_tag = kwargs['tags'] if 'tags' in kwargs else None self.args.workload_params = kwargs['workload_params'] if 'workload_params' in kwargs else None self.args.telemetry = kwargs['telemetry'] if 'telemetry' in kwargs else None @@ -40,70 +40,47 @@ def setUp(self, **kwargs: Any) -> None: def test_execute_default(self, mock_check_call: Mock) -> None: self.args.insecure = True - mock_check_call.return_value = 0 mock_convert = MagicMock() - - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - with patch.object(BenchmarkTestSuite, 'convert', mock_convert): - test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.execute() - mock_temp_directory.assert_called_once() - mock_check_call.assert_called_with( - f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/. /mock/temp/dir", cwd=os.getcwd(), shell=True) - mock_convert.assert_called_once_with('/mock/test_execution.json') - - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' - f'--workload=nyc_taxis --pipeline=benchmark-only --target-hosts=abc.com:80 --client-options="timeout:300"') + with patch.object(BenchmarkTestSuite, 'convert', mock_convert): + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' + f'--workload=nyc_taxis --pipeline=benchmark-only --target-hosts=abc.com:80 --client-options="timeout:300"') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') def test_execute_security_enabled_version_212_or_greater(self, mock_convert: Mock, mock_check_call: Mock) -> None: mock_check_call.return_value = 0 self.args.insecure = False - - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, self.password) - test_suite.execute() - mock_temp_directory.assert_called_once() - mock_convert.assert_called_once_with('/mock/test_execution.json') - mock_check_call.assert_called_with( - f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/. /mock/temp/dir", cwd=os.getcwd(), shell=True) - - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, - f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test' - f' --workload=nyc_taxis --pipeline=benchmark-only ' - f'--target-hosts=abc.com:443 ' - f'--client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'myStrongPassword123!\'"') + test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, self.password) + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test' + f' --workload=nyc_taxis --pipeline=benchmark-only ' + f'--target-hosts=abc.com:443 ' + f'--client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'myStrongPassword123!\'"') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') def test_execute_security_enabled(self, mock_convert: Mock, mock_check_call: Mock) -> None: mock_check_call.return_value = 0 self.args.insecure = True - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, "admin") - test_suite.execute() - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, - f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis --pipeline=benchmark-only ' - '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' - 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') + test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, "admin") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis --pipeline=benchmark-only ' + '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' + 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -114,23 +91,20 @@ def test_execute_default_with_optional_args(self, mock_convert: Mock, mock_check TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", telemetry=['node-stats'], telemetry_params="{\"example_key\":\"example_value\"}") - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.execute() - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com:80 ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--user-tag="key1:value1,key2:value2" --telemetry node-stats, --telemetry-params \'{"example_key":"example_value"}\' ' - '--client-options="timeout:300"') + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--user-tag="key1:value1,key2:value2" --telemetry node-stats, --telemetry-params \'{"example_key":"example_value"}\' ' + '--client-options="timeout:300"') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -138,22 +112,20 @@ def test_execute_default_with_no_telemetry_params(self, mock_convert: Mock, mock mock_check_call.return_value = 0 TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", telemetry=['node-stats', 'test']) - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.execute() - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com:80 ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, ' - '--client-options="timeout:300"') + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--user-tag="key1:value1,key2:value2" --telemetry node-stats,test, ' + '--client-options="timeout:300"') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -162,23 +134,21 @@ def test_execute_with_test_procedure_params(self, mock_convert: Mock, mock_check self.args.insecure = True TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", test_procedure="test-proc1,test-proc2") - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.execute() - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com:80 ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--test-procedure="test-proc1,test-proc2" ' - '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--test-procedure="test-proc1,test-proc2" ' + '--user-tag="key1:value1,key2:value2" ' + '--client-options="timeout:300"') @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') @@ -188,52 +158,64 @@ def test_execute_with_include_exclude_params(self, mock_convert: Mock, mock_chec TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", include_tasks="task1,type:index", exclude_tasks="task2,type:search") - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.execute() - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com:80 ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--exclude-tasks="task2,type:search" ' - '--include-tasks="task1,type:index" ' - '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--exclude-tasks="task2,type:search" ' + '--include-tasks="task1,type:index" ' + '--user-tag="key1:value1,key2:value2" ' + '--client-options="timeout:300"') - @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') - def test_execute_with_all_benchmark_optional_params(self, mock_convert: Mock, mock_check_call: Mock) -> None: - mock_check_call.return_value = 0 + def test_execute_with_all_benchmark_optional_params(self, mock_convert: Mock) -> None: self.args.insecure = True TestBenchmarkTestSuite.setUp(self, config="/home/test/benchmark.ini", tags="key1:value1,key2:value2", workload_params="{\"number_of_replicas\":\"1\"}", test_procedure="test-proc1,test-proc2", include_tasks="task1,type:index", exclude_tasks="task2,type:search") - with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: - mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() - mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' - with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: - mock_glob.return_value = ['/mock/test_execution.json'] - test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.execute() - self.assertEqual(mock_check_call.call_count, 2) - self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' - '/opensearch-benchmark/.benchmark/benchmark.ini ' - 'opensearchproject/opensearch-benchmark:latest execute-test ' - '--workload=nyc_taxis ' - '--pipeline=benchmark-only --target-hosts=abc.com:80 ' - '--workload-params \'{"number_of_replicas":"1"}\' ' - '--test-procedure="test-proc1,test-proc2" ' - '--exclude-tasks="task2,type:search" ' - '--include-tasks="task1,type:index" ' - '--user-tag="key1:value1,key2:value2" ' - '--client-options="timeout:300"') + with patch("subprocess.check_call") as mock_check_call: + test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, f'docker run --name docker-container-{test_suite.args.stack_suffix} -v /home/test/benchmark.ini:' + '/opensearch-benchmark/.benchmark/benchmark.ini ' + 'opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis ' + '--pipeline=benchmark-only --target-hosts=abc.com:80 ' + '--workload-params \'{"number_of_replicas":"1"}\' ' + '--test-procedure="test-proc1,test-proc2" ' + '--exclude-tasks="task2,type:search" ' + '--include-tasks="task1,type:index" ' + '--user-tag="key1:value1,key2:value2" ' + '--client-options="timeout:300"') + + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + @patch('test_workflow.benchmark_test.benchmark_test_suite.BenchmarkTestSuite.convert') + def test_execute_cluster_endpoint(self, mock_convert: Mock, mock_check_call: Mock) -> None: + mock_check_call.return_value = 0 + self.args.cluster_endpoint = "abc.com" + self.args.insecure = True + test_suite = BenchmarkTestSuite("abc.com:443", True, self.args, "admin") + test_suite.execute() + self.assertEqual(mock_check_call.call_count, 2) + self.assertEqual(mock_convert.call_count, 1) + mock_check_call.assert_called_with( + f"docker rm docker-container-{test_suite.args.stack_suffix}", cwd=os.getcwd(), shell=True) + self.assertEqual(test_suite.command, + f'docker run --name docker-container-{test_suite.args.stack_suffix} opensearchproject/opensearch-benchmark:latest execute-test ' + '--workload=nyc_taxis --pipeline=benchmark-only ' + '--target-hosts=abc.com:443 --client-options="timeout:300,use_ssl:true,' + 'verify_certs:false,basic_auth_user:\'admin\',basic_auth_password:\'admin\'"') @patch('pandas.json_normalize') @patch('pandas.read_csv') @@ -241,8 +223,10 @@ def test_execute_with_all_benchmark_optional_params(self, mock_convert: Mock, mo @patch('builtins.open') @patch('logging.info') @patch('shutil.get_terminal_size') - def test_convert(self, mock_get_terminal_size: Mock, mock_logging_info: Mock, mock_open: Mock, mock_json_load: Mock, mock_read_csv: Mock, + @patch('test_workflow.benchmark_test.benchmark_test_suite.subprocess.check_call') + def test_convert(self, mock_check_call: Mock, mock_get_terminal_size: Mock, mock_logging_info: Mock, mock_open: Mock, mock_json_load: Mock, mock_read_csv: Mock, mock_json_normalize: Mock) -> None: + self.args.cluster_endpoint = "abc.com" mock_get_terminal_size.return_value = MagicMock(columns=80) mock_open.return_value = MagicMock() mock_json_load.return_value = {"results": {"op_metrics": [{"metric": "value"}]}} @@ -250,10 +234,16 @@ def test_convert(self, mock_get_terminal_size: Mock, mock_logging_info: Mock, mo mock_read_csv.return_value = MagicMock() test_suite = BenchmarkTestSuite("abc.com:80", False, self.args, "") - test_suite.convert("mock_results.json") - - mock_open.assert_called_once_with("mock_results.json") - mock_json_load.assert_called_once() - mock_json_normalize.assert_called_once() - mock_read_csv.assert_called_once() - mock_logging_info.assert_called() + with patch('test_workflow.benchmark_test.benchmark_test_suite.TemporaryDirectory') as mock_temp_directory: + mock_temp_directory.return_value.__enter__.return_value.name = tempfile.gettempdir() + mock_temp_directory.return_value.__enter__.return_value.path = '/mock/temp/dir' + with patch('test_workflow.benchmark_test.benchmark_test_suite.glob.glob') as mock_glob: + mock_glob.return_value = ['/mock/test_execution.json'] + test_suite.convert() + mock_temp_directory.assert_called_once() + mock_check_call.assert_called_with(f"docker cp docker-container-{test_suite.args.stack_suffix}:opensearch-benchmark/. /mock/temp/dir", cwd=os.getcwd(), shell=True) + mock_open.assert_called_once_with("/mock/test_execution.json") + mock_json_load.assert_called_once() + mock_json_normalize.assert_called_once() + mock_read_csv.assert_called_once() + mock_logging_info.assert_called()