From 5578cc02aecfeaffdacf0308c2e7744734476497 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Fri, 8 Jun 2018 17:06:43 +0100 Subject: [PATCH 1/5] Add BenchmarkSuite.__str__ for debugging Signed-off-by: Stefan Marr --- rebench/model/benchmark_suite.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rebench/model/benchmark_suite.py b/rebench/model/benchmark_suite.py index ef8f6d65..c9b537d2 100644 --- a/rebench/model/benchmark_suite.py +++ b/rebench/model/benchmark_suite.py @@ -98,3 +98,6 @@ def max_runtime(self): def has_max_runtime(self): return self._max_runtime != -1 + + def __str__(self): + return "Suite(%s, %s)" % (self._name, self._command) From 5b17c80fba3d9a7f2eda34e3d5d784ff5eac6d56 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Fri, 8 Jun 2018 17:06:58 +0100 Subject: [PATCH 2/5] Fix processing of experiment filters Signed-off-by: Stefan Marr --- rebench/rebench.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rebench/rebench.py b/rebench/rebench.py index 29fabbce..e5acc602 100755 --- a/rebench/rebench.py +++ b/rebench/rebench.py @@ -163,9 +163,9 @@ def run(self, argv = None): # interpret remaining args exp_filter = args.exp_filter - exp_name = exp_filter[1] if len(exp_filter) > 1 and ( - not exp_filter[1].startswith("vm:") and - not exp_filter[1].startswith("s:")) else "all" + exp_name = exp_filter[0] if len(exp_filter) > 0 and ( + not exp_filter[0].startswith("vm:") and + not exp_filter[0].startswith("s:")) else "all" exp_filter = [f for f in exp_filter if (f.startswith("vm:") or f.startswith("s:"))] From 4458da3ef9e3b4cf775288a91e724b96cffac47e Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Fri, 8 Jun 2018 17:09:33 +0100 Subject: [PATCH 3/5] Bump version number Signed-off-by: Stefan Marr --- rebench/rebench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebench/rebench.py b/rebench/rebench.py index e5acc602..6130efb5 100755 --- a/rebench/rebench.py +++ b/rebench/rebench.py @@ -42,7 +42,7 @@ class ReBench: def __init__(self): - self.version = "0.10.0" + self.version = "0.10.1" self.options = None self._config = None From 43d4fe3df9bc041be13ab11ef89f520e499eada7 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Fri, 8 Jun 2018 19:08:55 +0100 Subject: [PATCH 4/5] Add tests for experiment name and filter handling Signed-off-by: Stefan Marr --- rebench/rebench.py | 17 +++++++----- rebench/tests/executor_test.py | 48 +++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/rebench/rebench.py b/rebench/rebench.py index 6130efb5..0a0e2b62 100755 --- a/rebench/rebench.py +++ b/rebench/rebench.py @@ -151,6 +151,15 @@ def shell_options(self): return parser + @staticmethod + def determine_exp_name_and_filters(filters): + exp_name = filters[0] if len(filters) > 0 and ( + not filters[0].startswith("vm:") and + not filters[0].startswith("s:")) else "all" + exp_filter = [f for f in filters if (f.startswith("vm:") or + f.startswith("s:"))] + return exp_name, exp_filter + def run(self, argv = None): if argv is None: argv = sys.argv @@ -161,13 +170,7 @@ def run(self, argv = None): cli_reporter = CliReporter(args.verbose) - # interpret remaining args - exp_filter = args.exp_filter - exp_name = exp_filter[0] if len(exp_filter) > 0 and ( - not exp_filter[0].startswith("vm:") and - not exp_filter[0].startswith("s:")) else "all" - exp_filter = [f for f in exp_filter if (f.startswith("vm:") or - f.startswith("s:"))] + exp_name, exp_filter = self.determine_exp_name_and_filters(args.exp_filter) try: config_filename = args.config[0] diff --git a/rebench/tests/executor_test.py b/rebench/tests/executor_test.py index 4593b077..04397c31 100644 --- a/rebench/tests/executor_test.py +++ b/rebench/tests/executor_test.py @@ -121,7 +121,52 @@ def test_basic_execution_with_magic_all(self): cnf = Configurator(self._path + '/small.conf', DataStore(), None, None, 'all', standard_data_file = self._tmp_file) self._basic_execution(cnf) - + + def test_shell_options_without_filters(self): + option_parser = ReBench().shell_options() + args = option_parser.parse_args(['-d', '-v', 'some.conf']) + self.assertEqual(args.exp_filter, []) + + def test_shell_options_with_filters(self): + option_parser = ReBench().shell_options() + args = option_parser.parse_args(['-d', '-v', 'some.conf', 'exp_name']) + self.assertEqual(args.exp_filter, ['exp_name']) + + def test_shell_options_with_vm_filter(self): + option_parser = ReBench().shell_options() + args = option_parser.parse_args(['-d', '-v', 'some.conf', 'vm:foo']) + self.assertEqual(args.exp_filter, ['vm:foo']) + + def test_determine_exp_name_and_filters_empty(self): + empty = [] + exp_name, exp_filter = ReBench.determine_exp_name_and_filters(empty) + self.assertEqual(exp_name, "all") + self.assertEqual(exp_filter, []) + + def test_determine_exp_name_and_filters_all(self): + filters = ['all'] + exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters) + self.assertEqual(exp_name, "all") + self.assertEqual(exp_filter, []) + + def test_determine_exp_name_and_filters_some_name(self): + filters = ['foo'] + exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters) + self.assertEqual(exp_name, "foo") + self.assertEqual(exp_filter, []) + + def test_determine_exp_name_and_filters_all_and_other(self): + filters = ['all', 'vm:bar', 's:b'] + exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters) + self.assertEqual(exp_name, "all") + self.assertEqual(exp_filter, ['vm:bar', 's:b']) + + def test_determine_exp_name_and_filters_only_others(self): + filters = ['vm:bar', 's:b'] + exp_name, exp_filter = ReBench.determine_exp_name_and_filters(filters) + self.assertEqual(exp_name, "all") + self.assertEqual(exp_filter, ['vm:bar', 's:b']) + def Popen_override(cmdline, stdout, stderr=None, shell=None): class Popen: @@ -154,5 +199,6 @@ def __exit__(self, _type, _value, _traceback): def test_suite(): return unittest.makeSuite(ExecutorTest) + if __name__ == "__main__": unittest.main(defaultTest='test_suite') From 6d610b19c36497ce8d8cf2e158efad35a694efb2 Mon Sep 17 00:00:00 2001 From: Stefan Marr Date: Fri, 8 Jun 2018 20:19:40 +0100 Subject: [PATCH 5/5] Fix error reporting for Codespeed Signed-off-by: Stefan Marr --- rebench/reporter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rebench/reporter.py b/rebench/reporter.py index 9b74ebff..1546282d 100644 --- a/rebench/reporter.py +++ b/rebench/reporter.py @@ -415,10 +415,10 @@ def _send_to_codespeed(self, results): "either a wrong URL in the config file, or an " "environment not configured in Codespeed. URL: " + self._cfg.url) - envs = set([i['environment'] for i in payload]) - projects = set([i['project'] for i in payload]) - benchmarks = set([i['benchmark'] for i in payload]) - executables = set([i['executable'] for i in payload]) + envs = list(set([i['environment'] for i in results])) + projects = list(set([i['project'] for i in results])) + benchmarks = list(set([i['benchmark'] for i in results])) + executables = list(set([i['executable'] for i in results])) logging.error("Sent data included environments: %s " "projects: %s benchmarks: %s executables: %s" % (envs, projects, benchmarks, executables))