diff --git a/benchmarks/scripts/cub/bench/config.py b/benchmarks/scripts/cub/bench/config.py index ee393ff12..eea9515ea 100644 --- a/benchmarks/scripts/cub/bench/config.py +++ b/benchmarks/scripts/cub/bench/config.py @@ -90,6 +90,21 @@ def __new__(cls, *args, **kwargs): cls._instance = super().__new__(cls, *args, **kwargs) cls._instance.ctk, cls._instance.cub, cls._instance.benchmarks = parse_meta() return cls._instance + + def label_to_variant_point(self, algname, label): + if label == "base": + return BasePoint() + + label_to_definition = {} + for param_space in self.benchmarks[algname]: + label_to_definition[param_space.label] = param_space.definition + + points = [] + for point in label.split('.'): + label, value = point.split('_') + points.append(RangePoint(label_to_definition[label], label, int(value))) + + return VariantPoint(points) def variant_space(self, algname): variants = [] diff --git a/benchmarks/scripts/verify.py b/benchmarks/scripts/verify.py new file mode 100755 index 000000000..f653fa4aa --- /dev/null +++ b/benchmarks/scripts/verify.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import cub.bench + + +def workload_header(ct_workload_space, rt_workload_space): + for ct_workload in ct_workload_space: + for rt_workload in rt_workload_space: + workload_point = ct_workload + rt_workload + return ", ".join([x.split('=')[0] for x in workload_point]) + + +def workload_entry(ct_workload, rt_workload): + workload_point = ct_workload + rt_workload + return ", ".join([x.split('=')[1] for x in workload_point]) + + +class VerifySeeker: + def __init__(self): + self.estimator = cub.bench.MedianCenterEstimator() + + def __call__(self, algname, ct_workload_space, rt_workload_space): + label = 'trp_0.ld_0.ipt_20.tpb_448.l2b_485.l2w_710' + variant_point = cub.bench.Config().label_to_variant_point(algname, label) + + print("{}, MinS, MedianS, MaxS".format(workload_header(ct_workload_space, rt_workload_space))) + for ct_workload in ct_workload_space: + bench = cub.bench.Bench(algname, variant_point, list(ct_workload)) + if bench.build(): + base = bench.get_base() + for rt_workload in rt_workload_space: + workload_point = ct_workload + rt_workload + base_samples, base_elapsed = base.do_run(workload_point, None) + variant_samples, _ = bench.do_run(workload_point, base_elapsed * 10) + min_speedup = min(base_samples) / min(variant_samples) + median_speedup = self.estimator(base_samples) / self.estimator(variant_samples) + max_speedup = max(base_samples) / max(variant_samples) + point_str = workload_entry(ct_workload, rt_workload) + print("{}, {}, {}, {}".format(point_str, min_speedup, median_speedup, max_speedup)) + + +def main(): + cub.bench.search(VerifySeeker()) + + +if __name__ == "__main__": + main()