diff --git a/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.template.yml b/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.template.yml index f4b18db9c6d..b3cd59e1b9f 100644 --- a/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.template.yml +++ b/.gitlab/benchmarks/bp-runner.microbenchmarks.fail-on-breach.template.yml @@ -7,6 +7,62 @@ experiments: run: fail_on_breach scenarios: # coreapiscenario + - name: coreapiscenario-has_listeners_no_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-has_listeners_with_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_no_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_1_listener + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_50_listeners + thresholds: + - execution_time < 0.05 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_no_args_no_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_no_args_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_exception_no_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_exception_listeners + thresholds: + - execution_time < 0.03 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_with_results_no_listeners + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_with_results_1_listener + thresholds: + - execution_time < 0.01 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_with_results_listeners + thresholds: + - execution_time < 0.05 ms + - max_rss_usage < 38.00 MB + - name: coreapiscenario-core_dispatch_with_results_50_listeners + thresholds: + - execution_time < 0.25 ms + - max_rss_usage < 38.00 MB - name: coreapiscenario-context_with_data_listeners thresholds: - execution_time < 0.02 ms diff --git a/benchmarks/core_api/config.yaml b/benchmarks/core_api/config.yaml index 16bd87f1b19..18fcd45a5a5 100644 --- a/benchmarks/core_api/config.yaml +++ b/benchmarks/core_api/config.yaml @@ -1,36 +1,108 @@ +# --- has_listeners microbenchmark --- +has_listeners_no_listeners: + listeners: 0 + set_item_count: 0 + get_item_exists: false + listener_raises: false +has_listeners_with_listeners: + listeners: 10 + set_item_count: 0 + get_item_exists: false + listener_raises: false + +# --- dispatch: listener count sweep --- core_dispatch_no_listeners: listeners: 0 set_item_count: 0 get_item_exists: false + listener_raises: false +core_dispatch_1_listener: + listeners: 1 + set_item_count: 0 + get_item_exists: false + listener_raises: false core_dispatch_listeners: listeners: 10 set_item_count: 0 get_item_exists: false + listener_raises: false +core_dispatch_50_listeners: + listeners: 50 + set_item_count: 0 + get_item_exists: false + listener_raises: false + +# --- dispatch: empty args tuple --- +core_dispatch_no_args_no_listeners: + listeners: 0 + set_item_count: 0 + get_item_exists: false + listener_raises: false +core_dispatch_no_args_listeners: + listeners: 10 + set_item_count: 0 + get_item_exists: false + listener_raises: false + +# --- dispatch: exception path (listeners raise, config._raise=False) --- +core_dispatch_exception_no_listeners: + listeners: 0 + set_item_count: 0 + get_item_exists: false + listener_raises: true +core_dispatch_exception_listeners: + listeners: 10 + set_item_count: 0 + get_item_exists: false + listener_raises: true + +# --- dispatch_with_results: listener count sweep --- core_dispatch_with_results_no_listeners: listeners: 0 set_item_count: 0 get_item_exists: false + listener_raises: false +core_dispatch_with_results_1_listener: + listeners: 1 + set_item_count: 0 + get_item_exists: false + listener_raises: false core_dispatch_with_results_listeners: listeners: 10 set_item_count: 0 get_item_exists: false + listener_raises: false +core_dispatch_with_results_50_listeners: + listeners: 50 + set_item_count: 0 + get_item_exists: false + listener_raises: false + +# --- context_with_data --- context_with_data_no_listeners: listeners: 0 set_item_count: 0 get_item_exists: false + listener_raises: false context_with_data_listeners: listeners: 10 set_item_count: 0 get_item_exists: false + listener_raises: false + +# --- context item get/set --- set_item: listeners: 0 set_item_count: 100 get_item_exists: false + listener_raises: false get_item_missing: listeners: 0 set_item_count: 0 get_item_exists: false + listener_raises: false get_item_exists: listeners: 0 set_item_count: 0 get_item_exists: true + listener_raises: false diff --git a/benchmarks/core_api/scenario.py b/benchmarks/core_api/scenario.py index 2d9f896b349..43faa2d32e8 100644 --- a/benchmarks/core_api/scenario.py +++ b/benchmarks/core_api/scenario.py @@ -13,13 +13,19 @@ class CoreAPIScenario(bm.Scenario): listeners: int set_item_count: int get_item_exists: bool + listener_raises: bool # whether registered listeners raise exceptions def run(self): - # Activate a number of no-op listeners for known events for _ in range(self.listeners): + if self.listener_raises: - def listener(*_): - pass + def listener(*_): + raise ValueError("benchmark listener exception") + + else: + + def listener(*_): + pass core.on(CUSTOM_EVENT_NAME, listener) core.on("context.started.with_data", listener) @@ -28,6 +34,16 @@ def listener(*_): if self.get_item_exists: core.set_item("key", "value") + def has_listeners(loops): + """Measure the cost of core.has_listeners""" + for _ in range(loops): + core.has_listeners(CUSTOM_EVENT_NAME) + + def core_dispatch_no_args(loops): + """Measure dispatch cost with an empty args tuple""" + for _ in range(loops): + core.dispatch(CUSTOM_EVENT_NAME, ()) + def core_dispatch(loops): """Measure the cost to dispatch an event on the hub""" for _ in range(loops): @@ -59,7 +75,11 @@ def get_item(loops): for _ in range(loops): core.find_item("key") - if "core_dispatch_with_results" in self.scenario_name: + if "has_listeners" in self.scenario_name: + yield has_listeners + elif "core_dispatch_no_args" in self.scenario_name: + yield core_dispatch_no_args + elif "core_dispatch_with_results" in self.scenario_name: yield core_dispatch_with_results elif "core_dispatch" in self.scenario_name: yield core_dispatch