From a16998d0699318ca98651fd9c22ef8e9ce96269d Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Thu, 27 Jul 2023 14:29:02 +0100 Subject: [PATCH 01/16] Make invocation number accessible from cmd line Create issue_216.conf Create issue_216_test.py Create issue_216.data give a value to invocation if in cmdline add functions to store and get the updated cmdline add invocation as a command --- docs/config.md | 3 +- rebench/executor.py | 3 ++ rebench/model/run_id.py | 8 +++++ rebench/tests/features/issue_216.conf | 39 ++++++++++++++++++++++++ rebench/tests/features/issue_216.data | 1 + rebench/tests/features/issue_216_test.py | 36 ++++++++++++++++++++++ 6 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 rebench/tests/features/issue_216.conf create mode 100644 rebench/tests/features/issue_216.data create mode 100644 rebench/tests/features/issue_216_test.py diff --git a/docs/config.md b/docs/config.md index c4bc2d3d..9c053fcd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -447,6 +447,7 @@ It supports various format variables, including: - executor (the executor's name) - input (the input variable's value) - iterations (the number of iterations) + - invocation (the current invocation) - suite (the name of the benchmark suite) - variable (another variable's value) - warmup (the number of iterations to be considered warmup iterations) @@ -458,7 +459,7 @@ Example: ```yaml benchmark_suites: ExampleSuite: - command: Harness %(benchmark)s --problem-size=%(input)s --iterations=%(iterations)s + command: Harness %(benchmark)s --problem-size=%(input)s --iterations=%(iterations)s %(invocations) ``` --- diff --git a/rebench/executor.py b/rebench/executor.py index c4a5baa2..065610e3 100644 --- a/rebench/executor.py +++ b/rebench/executor.py @@ -495,6 +495,9 @@ def _generate_data_point(self, cmdline, gauge_adapter, run_id, assert not self._print_execution_plan # execute the external program here output = "" + if 'invocation ' in cmdline: + cmdline = cmdline % {'invocation':run_id.completed_invocations +1} + run_id.store_cmdline(cmdline) try: self.ui.debug_output_info("{ind}Starting run\n", run_id, cmdline) diff --git a/rebench/model/run_id.py b/rebench/model/run_id.py index 06b06f73..a527b70c 100644 --- a/rebench/model/run_id.py +++ b/rebench/model/run_id.py @@ -47,6 +47,7 @@ def __init__(self, benchmark, cores, input_size, var_value, machine): self.executable = None self.executable_missing = False self.is_failed = True + self.updated_cmdline = [] self._max_invocation = 0 @@ -249,6 +250,7 @@ def _expand_vars(self, string): 'executor': self.benchmark.suite.executor.name, 'input': self.input_size_as_str, 'iterations': self.iterations, + 'invocation': '%(invocation)s', 'suite': self.benchmark.suite.name, 'variable': self.var_value_as_str, 'machine': self.machine_as_str, @@ -272,6 +274,12 @@ def cmdline(self): return self._cmdline return self._construct_cmdline() + def store_cmdline(self, cmd): + self.updated_cmdline.append(cmd) + + def get_cmdline(self): + return self.updated_cmdline + def _construct_cmdline(self): cmdline = "" if self.benchmark.suite.executor.path: diff --git a/rebench/tests/features/issue_216.conf b/rebench/tests/features/issue_216.conf new file mode 100644 index 00000000..282b1dca --- /dev/null +++ b/rebench/tests/features/issue_216.conf @@ -0,0 +1,39 @@ +# Config file for ReBench +# Config format is YAML (see http://yaml.org/ for detailed spec) + +# this run definition will be chosen if no parameters are given to rebench.py +default_experiment: Test +default_data_file: 'issue_216.data' + +# general configuration for runs +runs: + invocations: 5 + retries_after_failure: 3 + +benchmark_suites: + Suite: + gauge_adapter: TestExecutor + command: TestBenchMarks %(benchmark)s %(input)s %(variable)s %(invocation)s + input_sizes: [2, 10] + variable_values: + - val1 + benchmarks: + - Bench1 + - Bench2 + +executors: + TestRunner1: + path: tests + executable: test-vm1.py %(cores)s + cores: [1] + TestRunner2: + path: tests + executable: test-vm2.py + +experiments: + Test: + suites: + - Suite + executions: + - TestRunner1 + - TestRunner2 diff --git a/rebench/tests/features/issue_216.data b/rebench/tests/features/issue_216.data new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rebench/tests/features/issue_216.data @@ -0,0 +1 @@ + diff --git a/rebench/tests/features/issue_216_test.py b/rebench/tests/features/issue_216_test.py new file mode 100644 index 00000000..21a919a7 --- /dev/null +++ b/rebench/tests/features/issue_216_test.py @@ -0,0 +1,36 @@ +import os + +from ..rebench_test_case import ReBenchTestCase +from ...executor import Executor +from ...configurator import Configurator, load_config +from ...persistence import DataStore + + +class RetrieveIvocationTest(ReBenchTestCase): + + def setUp(self): + super(RetrieveIvocationTest, self).setUp() + os.chdir(self._path + '/../') + + def test_get_invocation_from_cmdline(self): + cnf = Configurator(load_config(os.path.join(self._path,'features', 'issue_216.conf')), + DataStore(self.ui), self.ui, None, + data_file=self._tmp_file) + runs = cnf.get_runs() + ex = Executor(runs, cnf.do_builds, self.ui) + ex.execute() + cmdlines = [run_id.get_cmdline() for run_id in ex.runs] + for cmd in cmdlines: + sequence_numbers = [self.get_sequence_number(cmdline) for cmdline in cmd] + is_increasing = self.is_list_increasing(sequence_numbers) + self.assertTrue(is_increasing) + + def is_list_increasing(self,lst): + return all(x < y for x, y in zip(lst, lst[1:])) + + def get_sequence_number(self,cmdline): + last_element = cmdline.split()[-1] + try: + return int(last_element) + except ValueError: + return float('-inf') From 314fed34aeb6b2f0311d809604a470dabda66f08 Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:49:21 +0100 Subject: [PATCH 02/16] fix typo Co-authored-by: Stefan Marr --- docs/config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index 9c053fcd..e60f40b0 100644 --- a/docs/config.md +++ b/docs/config.md @@ -459,7 +459,7 @@ Example: ```yaml benchmark_suites: ExampleSuite: - command: Harness %(benchmark)s --problem-size=%(input)s --iterations=%(iterations)s %(invocations) + command: Harness %(benchmark)s --problem-size=%(input)s --iterations=%(iterations)s %(invocation)s ``` --- From 89e3fa3b389ffeb1c368930eb9ceb272376d8fa7 Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:49:51 +0100 Subject: [PATCH 03/16] remove storage of cmdline Co-authored-by: Stefan Marr --- rebench/executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rebench/executor.py b/rebench/executor.py index 065610e3..175d7b1a 100644 --- a/rebench/executor.py +++ b/rebench/executor.py @@ -497,7 +497,6 @@ def _generate_data_point(self, cmdline, gauge_adapter, run_id, output = "" if 'invocation ' in cmdline: cmdline = cmdline % {'invocation':run_id.completed_invocations +1} - run_id.store_cmdline(cmdline) try: self.ui.debug_output_info("{ind}Starting run\n", run_id, cmdline) From 2a75661bec7753da391d0403df3290bcdbf6682d Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:50:17 +0100 Subject: [PATCH 04/16] remove cmdline list Co-authored-by: Stefan Marr --- rebench/model/run_id.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rebench/model/run_id.py b/rebench/model/run_id.py index a527b70c..452455f3 100644 --- a/rebench/model/run_id.py +++ b/rebench/model/run_id.py @@ -47,7 +47,6 @@ def __init__(self, benchmark, cores, input_size, var_value, machine): self.executable = None self.executable_missing = False self.is_failed = True - self.updated_cmdline = [] self._max_invocation = 0 From 337717a84ebdee5f889561e57e2439a392212a3c Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:50:37 +0100 Subject: [PATCH 05/16] remove methods only added for testing Co-authored-by: Stefan Marr --- rebench/model/run_id.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/rebench/model/run_id.py b/rebench/model/run_id.py index 452455f3..0a103934 100644 --- a/rebench/model/run_id.py +++ b/rebench/model/run_id.py @@ -273,12 +273,6 @@ def cmdline(self): return self._cmdline return self._construct_cmdline() - def store_cmdline(self, cmd): - self.updated_cmdline.append(cmd) - - def get_cmdline(self): - return self.updated_cmdline - def _construct_cmdline(self): cmdline = "" if self.benchmark.suite.executor.path: From 1cb91d40f7f80573522ad7461f77331f0ce0310c Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:51:49 +0100 Subject: [PATCH 06/16] Delete issue_216.data --- rebench/tests/features/issue_216.data | 1 - 1 file changed, 1 deletion(-) delete mode 100644 rebench/tests/features/issue_216.data diff --git a/rebench/tests/features/issue_216.data b/rebench/tests/features/issue_216.data deleted file mode 100644 index 8b137891..00000000 --- a/rebench/tests/features/issue_216.data +++ /dev/null @@ -1 +0,0 @@ - From 86b2d990455a2ccd768e206fdf0adabbbfc8b56c Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Fri, 28 Jul 2023 14:03:47 +0100 Subject: [PATCH 07/16] get rid of pointless if statement --- rebench/executor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rebench/executor.py b/rebench/executor.py index 175d7b1a..9d78da3e 100644 --- a/rebench/executor.py +++ b/rebench/executor.py @@ -495,8 +495,7 @@ def _generate_data_point(self, cmdline, gauge_adapter, run_id, assert not self._print_execution_plan # execute the external program here output = "" - if 'invocation ' in cmdline: - cmdline = cmdline % {'invocation':run_id.completed_invocations +1} + cmdline = cmdline % {'invocation':run_id.completed_invocations +1} try: self.ui.debug_output_info("{ind}Starting run\n", run_id, cmdline) From 8e1d0967e20829499164747698715c274607762e Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Mon, 31 Jul 2023 16:57:35 +0100 Subject: [PATCH 08/16] Update to follow same style as issue_31 --- rebench/tests/features/issue_216_test.py | 68 +++++++++++++++--------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/rebench/tests/features/issue_216_test.py b/rebench/tests/features/issue_216_test.py index 21a919a7..01dbe80d 100644 --- a/rebench/tests/features/issue_216_test.py +++ b/rebench/tests/features/issue_216_test.py @@ -1,36 +1,52 @@ -import os - +# Copyright (c) 2014 Tobias Pape +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from ...configurator import Configurator, load_config +from ...executor import Executor +from ...persistence import DataStore +from ..persistence import TestPersistence from ..rebench_test_case import ReBenchTestCase -from ...executor import Executor -from ...configurator import Configurator, load_config -from ...persistence import DataStore -class RetrieveIvocationTest(ReBenchTestCase): +class Issue216CurrentInvocation(ReBenchTestCase): def setUp(self): - super(RetrieveIvocationTest, self).setUp() - os.chdir(self._path + '/../') + super(Issue216CurrentInvocation, self).setUp() + self._set_path(__file__) - def test_get_invocation_from_cmdline(self): - cnf = Configurator(load_config(os.path.join(self._path,'features', 'issue_216.conf')), - DataStore(self.ui), self.ui, None, + def _records_data_points(self, exp_name, num_data_points): + cnf = Configurator(load_config(self._path + '/issue_216.conf'), DataStore(self.ui), + self.ui, exp_name=exp_name, data_file=self._tmp_file) runs = cnf.get_runs() - ex = Executor(runs, cnf.do_builds, self.ui) - ex.execute() - cmdlines = [run_id.get_cmdline() for run_id in ex.runs] - for cmd in cmdlines: - sequence_numbers = [self.get_sequence_number(cmdline) for cmdline in cmd] - is_increasing = self.is_list_increasing(sequence_numbers) - self.assertTrue(is_increasing) - def is_list_increasing(self,lst): - return all(x < y for x, y in zip(lst, lst[1:])) + persistence = TestPersistence() + persistence.use_on(runs) + ex = Executor(runs, False, self.ui) + ex.execute() + self.assertEqual(1, len(cnf.get_runs())) + run = next(iter(cnf.get_runs())) + self.assertEqual(num_data_points, run.get_number_of_data_points()) + return persistence.get_data_points() - def get_sequence_number(self,cmdline): - last_element = cmdline.split()[-1] - try: - return int(last_element) - except ValueError: - return float('-inf') + def test_associates_measurements_and_data_points_correctly(self): + data_points = self._records_data_points('Test1', 10) + for point, i in zip(data_points, range(10)): + for measurement in point.get_measurements(): + self.assertEqual(i, int(measurement.value)) From 3d2e0053b59bf5217f048d02352b90321e450b25 Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Mon, 31 Jul 2023 16:59:04 +0100 Subject: [PATCH 09/16] update to work with new test --- rebench/tests/features/issue_216.conf | 37 ++++++++------------------- 1 file changed, 10 insertions(+), 27 deletions(-) diff --git a/rebench/tests/features/issue_216.conf b/rebench/tests/features/issue_216.conf index 282b1dca..4528bea8 100644 --- a/rebench/tests/features/issue_216.conf +++ b/rebench/tests/features/issue_216.conf @@ -1,39 +1,22 @@ -# Config file for ReBench -# Config format is YAML (see http://yaml.org/ for detailed spec) - -# this run definition will be chosen if no parameters are given to rebench.py default_experiment: Test -default_data_file: 'issue_216.data' - -# general configuration for runs -runs: - invocations: 5 - retries_after_failure: 3 benchmark_suites: Suite: - gauge_adapter: TestExecutor - command: TestBenchMarks %(benchmark)s %(input)s %(variable)s %(invocation)s - input_sizes: [2, 10] - variable_values: - - val1 + gauge_adapter: Multivariate + command: TestBenchMarks %(benchmark)s %(invocation)s benchmarks: - - Bench1 - - Bench2 + - Bench1 executors: - TestRunner1: - path: tests - executable: test-vm1.py %(cores)s - cores: [1] - TestRunner2: - path: tests - executable: test-vm2.py + TestRunner: + path: . + executable: issue_216_vm.py experiments: - Test: + Test1: suites: - Suite executions: - - TestRunner1 - - TestRunner2 + - TestRunner: + input_sizes: + - 10 From 7938e715658107caf1b20f5b67e5502b006ebab4 Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Mon, 31 Jul 2023 17:00:05 +0100 Subject: [PATCH 10/16] create vm file to get invocation for issue 216 --- rebench/tests/features/issue_216_vm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 rebench/tests/features/issue_216_vm.py diff --git a/rebench/tests/features/issue_216_vm.py b/rebench/tests/features/issue_216_vm.py new file mode 100644 index 00000000..c131ddf9 --- /dev/null +++ b/rebench/tests/features/issue_216_vm.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# simple script emulating an executor generating benchmark results +import sys + +print(sys.argv) + +print("Harness Name:", sys.argv[1]) +print("Bench Name:", sys.argv[2]) +print("Current Invocation:", sys.argv[3]) + +INVOCATION_NUM = int(sys.argv[3]) +for i in range(0, INVOCATION_NUM): + print("%d:RESULT-total: %d.%d" % (i, i, i)) From b73bcf0f4936af946754960e409e0ec3da5d1e2e Mon Sep 17 00:00:00 2001 From: naomiGrew <97107839+naomiGrew@users.noreply.github.com> Date: Mon, 31 Jul 2023 17:01:37 +0100 Subject: [PATCH 11/16] add description of new invocation feature --- docs/config.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/config.md b/docs/config.md index e60f40b0..48bec2c8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -206,7 +206,11 @@ the experiment in general. **invocations:** -The number of times an executor is executed for a given run. +The number of times an executor is executed for a given run +The current invocation can be retrieved using a benchmark harness. + +The invocations setting can be used e.g can be used as an identifier when storing the results, +allowing you to distinguish between different runs of a benchmark with different tools. Default: `1` From 5dc8e9bdab25b884c30ef70f6dc3a89537f2d005 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Tue, 1 Aug 2023 13:21:26 +0100 Subject: [PATCH 12/16] Add example to documentation and link things up --- docs/config.md | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/docs/config.md b/docs/config.md index 48bec2c8..79a62554 100644 --- a/docs/config.md +++ b/docs/config.md @@ -204,13 +204,11 @@ All keys that can be used in the `runs` mapping can also be used for the definition of a benchmark, benchmark suite, executor, a concrete experiment, and the experiment in general. -**invocations:** + -The number of times an executor is executed for a given run -The current invocation can be retrieved using a benchmark harness. +**invocations:** -The invocations setting can be used e.g can be used as an identifier when storing the results, -allowing you to distinguish between different runs of a benchmark with different tools. +The number of times an executor is executed for a given run. Default: `1` @@ -221,6 +219,35 @@ runs: invocations: 100 ``` +*Access to the current invocation number:* + +The current invocation number can be used similar to [other variables](#format-vars) +as `%(invocation)s` in any configuration part that is used to compose the command line +for a run. + +This can be useful, for instance when one needs to generate different output files +for each invocation. + +Example: + +```yaml +benchmark_suites: + ExampleSuite: + invocations: 3 + command: Harness -output=%(benchmark)-%(invocation)s.log %(benchmark) + benchmark: + - Benchmark1 +``` + +The above example will execute the `Benchmark1` three times, each with a different +output file name: + +```bash +Harness -output=Benchmark1-1.log Benchmark1 +Harness -output=Benchmark1-2.log Benchmark1 +Harness -output=Benchmark1-3.log Benchmark1 +``` + --- **iterations:** @@ -438,6 +465,8 @@ benchmark_suites: --- + + **command:** The command for the benchmark harness. It will be combined with the @@ -451,7 +480,7 @@ It supports various format variables, including: - executor (the executor's name) - input (the input variable's value) - iterations (the number of iterations) - - invocation (the current invocation) + - [invocation](#invocations) (the current invocation) - suite (the name of the benchmark suite) - variable (another variable's value) - warmup (the number of iterations to be considered warmup iterations) From 738f2c366aa082e9c86b0a580fe9064898763a3d Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Tue, 1 Aug 2023 13:25:54 +0100 Subject: [PATCH 13/16] Add a comment --- rebench/executor.py | 6 ++++-- rebench/model/run_id.py | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rebench/executor.py b/rebench/executor.py index 9d78da3e..8c3273b8 100644 --- a/rebench/executor.py +++ b/rebench/executor.py @@ -493,9 +493,11 @@ def _get_gauge_adapter_instance(self, run_id): def _generate_data_point(self, cmdline, gauge_adapter, run_id, termination_check): assert not self._print_execution_plan - # execute the external program here output = "" - cmdline = cmdline % {'invocation':run_id.completed_invocations +1} + + # replace the invocation number in the command line + cmdline = cmdline % {'invocation': run_id.completed_invocations + 1} + try: self.ui.debug_output_info("{ind}Starting run\n", run_id, cmdline) diff --git a/rebench/model/run_id.py b/rebench/model/run_id.py index 0a103934..35894ea2 100644 --- a/rebench/model/run_id.py +++ b/rebench/model/run_id.py @@ -249,6 +249,9 @@ def _expand_vars(self, string): 'executor': self.benchmark.suite.executor.name, 'input': self.input_size_as_str, 'iterations': self.iterations, + + # the invocation number needs to be set right before execution + # we don't know it here, and it would change the RunId identity 'invocation': '%(invocation)s', 'suite': self.benchmark.suite.name, 'variable': self.var_value_as_str, From fa2064284e80890b0e8051de8bcd5ebf55d2e8a7 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Tue, 1 Aug 2023 14:03:14 +0100 Subject: [PATCH 14/16] Simplify test to really focus on the invocation number --- rebench/tests/features/issue_216.conf | 9 ++++----- rebench/tests/features/issue_216_test.py | 15 ++++++++------- rebench/tests/features/issue_216_vm.py | 12 ++---------- 3 files changed, 14 insertions(+), 22 deletions(-) mode change 100644 => 100755 rebench/tests/features/issue_216_vm.py diff --git a/rebench/tests/features/issue_216.conf b/rebench/tests/features/issue_216.conf index 4528bea8..1f9f0855 100644 --- a/rebench/tests/features/issue_216.conf +++ b/rebench/tests/features/issue_216.conf @@ -3,7 +3,8 @@ default_experiment: Test benchmark_suites: Suite: gauge_adapter: Multivariate - command: TestBenchMarks %(benchmark)s %(invocation)s + command: TestBenchMarks %(benchmark)s %(invocation)s + invocations: 4 benchmarks: - Bench1 @@ -13,10 +14,8 @@ executors: executable: issue_216_vm.py experiments: - Test1: + Test: suites: - Suite executions: - - TestRunner: - input_sizes: - - 10 + - TestRunner diff --git a/rebench/tests/features/issue_216_test.py b/rebench/tests/features/issue_216_test.py index 01dbe80d..882f582d 100644 --- a/rebench/tests/features/issue_216_test.py +++ b/rebench/tests/features/issue_216_test.py @@ -34,19 +34,20 @@ def _records_data_points(self, exp_name, num_data_points): cnf = Configurator(load_config(self._path + '/issue_216.conf'), DataStore(self.ui), self.ui, exp_name=exp_name, data_file=self._tmp_file) - runs = cnf.get_runs() + runs = list(cnf.get_runs()) + self.assertEqual(1, len(runs)) persistence = TestPersistence() persistence.use_on(runs) ex = Executor(runs, False, self.ui) ex.execute() - self.assertEqual(1, len(cnf.get_runs())) - run = next(iter(cnf.get_runs())) + + run = runs[0] self.assertEqual(num_data_points, run.get_number_of_data_points()) return persistence.get_data_points() def test_associates_measurements_and_data_points_correctly(self): - data_points = self._records_data_points('Test1', 10) - for point, i in zip(data_points, range(10)): - for measurement in point.get_measurements(): - self.assertEqual(i, int(measurement.value)) + data_points = self._records_data_points('Test', 4) + for point, i in zip(data_points, range(4)): + self.assertEqual(1, len(point.get_measurements())) + self.assertEqual(i + 1, int(point.get_measurements()[0].value)) diff --git a/rebench/tests/features/issue_216_vm.py b/rebench/tests/features/issue_216_vm.py old mode 100644 new mode 100755 index c131ddf9..f32320c6 --- a/rebench/tests/features/issue_216_vm.py +++ b/rebench/tests/features/issue_216_vm.py @@ -1,13 +1,5 @@ #!/usr/bin/env python3 -# simple script emulating an executor generating benchmark results import sys -print(sys.argv) - -print("Harness Name:", sys.argv[1]) -print("Bench Name:", sys.argv[2]) -print("Current Invocation:", sys.argv[3]) - -INVOCATION_NUM = int(sys.argv[3]) -for i in range(0, INVOCATION_NUM): - print("%d:RESULT-total: %d.%d" % (i, i, i)) +invocation_number = int(sys.argv[3]) +print("RESULT-total: %d" % invocation_number) From 3a8ac313388212af9efe6d2e511e6b3ce5b7f92a Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Tue, 1 Aug 2023 14:18:23 +0100 Subject: [PATCH 15/16] Need to replace invocation number in command line before having the any adapter applied The adapters may introduce arbitrary data, for instance the TimeAdapter may introduces a string that encodes how to output MaxRSS using %M. Since all this can be confused with our variables, we need to use a new `. cmdline_for_next_invocation()` method when aquiring the command. --- rebench/executor.py | 3 --- rebench/interop/adapter.py | 4 ++-- rebench/interop/perf_adapter.py | 3 ++- rebench/interop/time_adapter.py | 4 ++-- rebench/model/run_id.py | 4 ++++ rebench/tests/interop/plain_seconds_log_adapter_test.py | 2 +- rebench/tests/interop/time_adapter_test.py | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/rebench/executor.py b/rebench/executor.py index 8c3273b8..f5fa83dd 100644 --- a/rebench/executor.py +++ b/rebench/executor.py @@ -495,9 +495,6 @@ def _generate_data_point(self, cmdline, gauge_adapter, run_id, assert not self._print_execution_plan output = "" - # replace the invocation number in the command line - cmdline = cmdline % {'invocation': run_id.completed_invocations + 1} - try: self.ui.debug_output_info("{ind}Starting run\n", run_id, cmdline) diff --git a/rebench/interop/adapter.py b/rebench/interop/adapter.py index f55ff572..2434cc98 100644 --- a/rebench/interop/adapter.py +++ b/rebench/interop/adapter.py @@ -40,14 +40,14 @@ def __init__(self, include_faulty, executor): self._executor = executor def acquire_command(self, run_id): - return run_id.cmdline() + return run_id.cmdline_for_next_invocation() def parse_data(self, data, run_id, invocation): raise NotImplementedError() def check_for_error(self, line): """Check whether the output line contains one of the common error - messages. If its an erroneous run, the result has to be discarded. + messages. If it's an erroneous run, the result has to be discarded. """ if self._include_faulty: return False diff --git a/rebench/interop/perf_adapter.py b/rebench/interop/perf_adapter.py index 82df0553..6d697088 100644 --- a/rebench/interop/perf_adapter.py +++ b/rebench/interop/perf_adapter.py @@ -19,4 +19,5 @@ def parse_data(self, data, run_id, invocation): def acquire_command(self, run_id): profiler = self._get_profiler(run_id) - return profiler.command + " " + profiler.record_args + " " + run_id.cmdline() + return (profiler.command + " " + profiler.record_args + " " + + run_id.cmdline_for_next_invocation()) diff --git a/rebench/interop/time_adapter.py b/rebench/interop/time_adapter.py index 1fb91248..65c27b67 100644 --- a/rebench/interop/time_adapter.py +++ b/rebench/interop/time_adapter.py @@ -55,7 +55,7 @@ def _create_command(self, command): return "/usr/bin/time -p %s" % command def acquire_command(self, run_id): - command = run_id.cmdline() + command = run_id.cmdline_for_next_invocation() if not self._completed_time_availability_check: self._check_which_time_command_is_available() @@ -147,4 +147,4 @@ class TimeManualAdapter(TimeAdapter): This is useful for runs on remote machines like the Tilera or ARM boards. """ def acquire_command(self, run_id): - return run_id.cmdline() + return run_id.cmdline_for_next_invocation() diff --git a/rebench/model/run_id.py b/rebench/model/run_id.py index 35894ea2..b880f146 100644 --- a/rebench/model/run_id.py +++ b/rebench/model/run_id.py @@ -276,6 +276,10 @@ def cmdline(self): return self._cmdline return self._construct_cmdline() + def cmdline_for_next_invocation(self): + """Replace the invocation number in the command line""" + return self.cmdline() % {'invocation': self.completed_invocations + 1} + def _construct_cmdline(self): cmdline = "" if self.benchmark.suite.executor.path: diff --git a/rebench/tests/interop/plain_seconds_log_adapter_test.py b/rebench/tests/interop/plain_seconds_log_adapter_test.py index fcce5d5a..91ee523a 100644 --- a/rebench/tests/interop/plain_seconds_log_adapter_test.py +++ b/rebench/tests/interop/plain_seconds_log_adapter_test.py @@ -27,7 +27,7 @@ class PlainSecondsAdapterTest(TestCase): def test_acquire_command(self): class _TestRunId(object): - def cmdline(self): + def cmdline_for_next_invocation(self): return "FOO" adapter = PlainSecondsLogAdapter(False, None) cmd = adapter.acquire_command(_TestRunId()) diff --git a/rebench/tests/interop/time_adapter_test.py b/rebench/tests/interop/time_adapter_test.py index 14e75f51..a73f964d 100644 --- a/rebench/tests/interop/time_adapter_test.py +++ b/rebench/tests/interop/time_adapter_test.py @@ -24,7 +24,7 @@ class _TestRunId(object): - def cmdline(self): + def cmdline_for_next_invocation(self): return "FOO" From 02b8fe97fc07eec33e7e5caa4769983888bc42d0 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Tue, 1 Aug 2023 14:39:07 +0100 Subject: [PATCH 16/16] Fix copyright --- rebench/tests/features/issue_216_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebench/tests/features/issue_216_test.py b/rebench/tests/features/issue_216_test.py index 882f582d..1a756d89 100644 --- a/rebench/tests/features/issue_216_test.py +++ b/rebench/tests/features/issue_216_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2014 Tobias Pape +# Copyright (c) 2023 Naomi Grew, Stefan Marr # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to