Make invocation number accessible (#230)

docs/config.md

@@ -204,6 +204,8 @@ All keys that can be used in the `runs` mapping can also be used for the
 definition of a benchmark, benchmark suite, executor, a concrete experiment, and
 the experiment in general.
 
+<a id="invocations"></a>
+
 **invocations:**
 
 The number of times an executor is executed for a given run.
@@ -217,6 +219,35 @@ runs:
   invocations: 100
 ```
 
+*Access to the current invocation number:*
+
+The current invocation number can be used similar to [other variables](#format-vars)
+as `%(invocation)s` in any configuration part that is used to compose the command line
+for a run.
+
+This can be useful, for instance when one needs to generate different output files
+for each invocation.
+
+Example:
+
+```yaml
+benchmark_suites:
+  ExampleSuite:
+    invocations: 3
+    command: Harness -output=%(benchmark)-%(invocation)s.log %(benchmark)
+    benchmark:
+      - Benchmark1
+```
+
+The above example will execute the `Benchmark1` three times, each with a different
+output file name:
+
+```bash
+Harness -output=Benchmark1-1.log Benchmark1
+Harness -output=Benchmark1-2.log Benchmark1
+Harness -output=Benchmark1-3.log Benchmark1
+```
+
 ---
 
 **iterations:**
@@ -434,6 +465,8 @@ benchmark_suites:
 
 ---
 
+<a id="format-vars"></a>
+
 **command:**
 
 The command for the benchmark harness. It will be combined with the
@@ -447,6 +480,7 @@ It supports various format variables, including:
  - executor (the executor's name)
  - input (the input variable's value)
  - iterations (the number of iterations)
+ - [invocation](#invocations) (the current invocation)
  - suite (the name of the benchmark suite)
  - variable (another variable's value)
  - warmup (the number of iterations to be considered warmup iterations)
@@ -458,7 +492,7 @@ Example:
 ```yaml
 benchmark_suites:
   ExampleSuite:
-    command: Harness %(benchmark)s --problem-size=%(input)s --iterations=%(iterations)s
+    command: Harness %(benchmark)s --problem-size=%(input)s --iterations=%(iterations)s %(invocation)s
 ```
 
 ---

rebench/executor.py

@@ -493,8 +493,8 @@ def _get_gauge_adapter_instance(self, run_id):
     def _generate_data_point(self, cmdline, gauge_adapter, run_id,
                              termination_check):
         assert not self._print_execution_plan
-        # execute the external program here
         output = ""
+
         try:
             self.ui.debug_output_info("{ind}Starting run\n", run_id, cmdline)
 

rebench/interop/adapter.py

@@ -40,14 +40,14 @@ def __init__(self, include_faulty, executor):
         self._executor = executor
 
     def acquire_command(self, run_id):
-        return run_id.cmdline()
+        return run_id.cmdline_for_next_invocation()
 
     def parse_data(self, data, run_id, invocation):
         raise NotImplementedError()
 
     def check_for_error(self, line):
         """Check whether the output line contains one of the common error
-           messages. If its an erroneous run, the result has to be discarded.
+           messages. If it's an erroneous run, the result has to be discarded.
         """
         if self._include_faulty:
             return False

rebench/interop/perf_adapter.py

@@ -19,4 +19,5 @@ def parse_data(self, data, run_id, invocation):
 
     def acquire_command(self, run_id):
         profiler = self._get_profiler(run_id)
-        return profiler.command + " " + profiler.record_args + " " + run_id.cmdline()
+        return (profiler.command + " " + profiler.record_args + " " +
+                run_id.cmdline_for_next_invocation())

rebench/interop/time_adapter.py

@@ -55,7 +55,7 @@ def _create_command(self, command):
             return "/usr/bin/time -p %s" % command
 
     def acquire_command(self, run_id):
-        command = run_id.cmdline()
+        command = run_id.cmdline_for_next_invocation()
 
         if not self._completed_time_availability_check:
             self._check_which_time_command_is_available()
@@ -147,4 +147,4 @@ class TimeManualAdapter(TimeAdapter):
        This is useful for runs on remote machines like the Tilera or ARM boards.
     """
     def acquire_command(self, run_id):
-        return run_id.cmdline()
+        return run_id.cmdline_for_next_invocation()

rebench/model/run_id.py

@@ -249,6 +249,10 @@ def _expand_vars(self, string):
                              'executor': self.benchmark.suite.executor.name,
                              'input': self.input_size_as_str,
                              'iterations': self.iterations,
+
+                             # the invocation number needs to be set right before execution
+                             # we don't know it here, and it would change the RunId identity
+                             'invocation': '%(invocation)s',
                              'suite': self.benchmark.suite.name,
                              'variable': self.var_value_as_str,
                              'machine': self.machine_as_str,
@@ -272,6 +276,10 @@ def cmdline(self):
             return self._cmdline
         return self._construct_cmdline()
 
+    def cmdline_for_next_invocation(self):
+        """Replace the invocation number in the command line"""
+        return self.cmdline() % {'invocation': self.completed_invocations + 1}
+
     def _construct_cmdline(self):
         cmdline = ""
         if self.benchmark.suite.executor.path:

rebench/tests/features/issue_216.conf

@@ -0,0 +1,21 @@
+default_experiment: Test
+
+benchmark_suites:
+    Suite:
+        gauge_adapter: Multivariate
+        command: TestBenchMarks %(benchmark)s %(invocation)s
+        invocations: 4
+        benchmarks:
+          - Bench1
+
+executors:
+    TestRunner:
+        path: .
+        executable: issue_216_vm.py
+
+experiments:
+    Test:
+        suites:
+            - Suite
+        executions:
+            - TestRunner

rebench/tests/features/issue_216_test.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2023 Naomi Grew, Stefan Marr
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+from ...configurator           import Configurator, load_config
+from ...executor               import Executor
+from ...persistence            import DataStore
+from ..persistence import TestPersistence
+from ..rebench_test_case import ReBenchTestCase
+
+
+class Issue216CurrentInvocation(ReBenchTestCase):
+
+    def setUp(self):
+        super(Issue216CurrentInvocation, self).setUp()
+        self._set_path(__file__)
+
+    def _records_data_points(self, exp_name, num_data_points):
+        cnf = Configurator(load_config(self._path + '/issue_216.conf'), DataStore(self.ui),
+                           self.ui, exp_name=exp_name,
+                           data_file=self._tmp_file)
+        runs = list(cnf.get_runs())
+        self.assertEqual(1, len(runs))
+
+        persistence = TestPersistence()
+        persistence.use_on(runs)
+        ex = Executor(runs, False, self.ui)
+        ex.execute()
+
+        run = runs[0]
+        self.assertEqual(num_data_points, run.get_number_of_data_points())
+        return persistence.get_data_points()
+
+    def test_associates_measurements_and_data_points_correctly(self):
+        data_points = self._records_data_points('Test', 4)
+        for point, i in zip(data_points, range(4)):
+            self.assertEqual(1, len(point.get_measurements()))
+            self.assertEqual(i + 1, int(point.get_measurements()[0].value))

rebench/tests/features/issue_216_vm.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+import sys
+
+invocation_number = int(sys.argv[3])
+print("RESULT-total:    %d" % invocation_number)

rebench/tests/interop/plain_seconds_log_adapter_test.py

@@ -27,7 +27,7 @@ class PlainSecondsAdapterTest(TestCase):
 
     def test_acquire_command(self):
         class _TestRunId(object):
-            def cmdline(self):
+            def cmdline_for_next_invocation(self):
                 return "FOO"
         adapter = PlainSecondsLogAdapter(False, None)
         cmd = adapter.acquire_command(_TestRunId())

rebench/tests/interop/time_adapter_test.py

@@ -24,7 +24,7 @@
 
 
 class _TestRunId(object):
-    def cmdline(self):
+    def cmdline_for_next_invocation(self):
         return "FOO"