Multi-Client Runner with pubsub-sub-bench Integration

Author: fcostaoliveira

redis_benchmarks_specification/__cli__/stats.py

@@ -123,6 +123,25 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                     )
                     test_result = False
 
+                # Validate client configuration format
+                has_clientconfig = "clientconfig" in benchmark_config
+                has_clientconfigs = "clientconfigs" in benchmark_config
+
+                if has_clientconfig and has_clientconfigs:
+                    logging.error(
+                        "Test {} has both 'clientconfig' and 'clientconfigs'. Only one format is allowed.".format(
+                            test_name
+                        )
+                    )
+                    test_result = False
+                elif not has_clientconfig and not has_clientconfigs:
+                    logging.error(
+                        "Test {} is missing client configuration. Must have either 'clientconfig' or 'clientconfigs'.".format(
+                            test_name
+                        )
+                    )
+                    test_result = False
+
                 test_names.append(test_name)
                 group = ""
                 is_memtier = False
@@ -150,12 +169,24 @@ def generate_stats_cli_command_logic(args, project_name, project_version):
                 for tested_command in origin_tested_commands:
                     tested_commands.append(tested_command.lower())
                 if is_memtier:
-                    arguments = benchmark_config["clientconfig"]["arguments"]
-                    arg_list = (
-                        benchmark_config["clientconfig"]["arguments"]
-                        .replace('"', "")
-                        .split()
-                    )
+                    # Handle both clientconfig and clientconfigs formats
+                    if "clientconfigs" in benchmark_config:
+                        # For multiple configs, use the first one for stats analysis
+                        # TODO: Consider aggregating stats from all configs
+                        arguments = benchmark_config["clientconfigs"][0]["arguments"]
+                        arg_list = (
+                            benchmark_config["clientconfigs"][0]["arguments"]
+                            .replace('"', "")
+                            .split()
+                        )
+                    else:
+                        # Legacy single clientconfig format
+                        arguments = benchmark_config["clientconfig"]["arguments"]
+                        arg_list = (
+                            benchmark_config["clientconfig"]["arguments"]
+                            .replace('"', "")
+                            .split()
+                        )
 
                     data_size = get_arg_value(arg_list, "--data-size", data_size)
                     data_size = get_arg_value(arg_list, "-d", data_size)

redis_benchmarks_specification/__common__/spec.py

@@ -54,9 +54,19 @@ def extract_redis_configuration_from_topology(topologies_map, topology_spec_name
 
 
 def extract_client_cpu_limit(benchmark_config):
-    db_cpu_limit = benchmark_config["clientconfig"]["resources"]["requests"]["cpus"]
-    ceil_db_cpu_limit = math.ceil(float(db_cpu_limit))
-    return ceil_db_cpu_limit
+    # Handle both clientconfig (single) and clientconfigs (multiple) formats
+    if "clientconfigs" in benchmark_config:
+        # For multiple configs, return the sum of all CPU limits
+        total_cpu_limit = 0
+        for client_config in benchmark_config["clientconfigs"]:
+            cpu_limit = client_config["resources"]["requests"]["cpus"]
+            total_cpu_limit += float(cpu_limit)
+        return math.ceil(total_cpu_limit)
+    else:
+        # Legacy single clientconfig format
+        db_cpu_limit = benchmark_config["clientconfig"]["resources"]["requests"]["cpus"]
+        ceil_db_cpu_limit = math.ceil(float(db_cpu_limit))
+        return ceil_db_cpu_limit
 
 
 def extract_build_variant_variations(benchmark_config, keyname="build-variants"):
@@ -74,9 +84,60 @@ def extract_client_container_image(benchmark_config, keyname="clientconfig"):
     return client_container_image
 
 
+def extract_client_container_images(benchmark_config):
+    """Extract container images for both single and multiple client configs"""
+    if "clientconfigs" in benchmark_config:
+        # Multiple client configs - return list of images
+        images = []
+        for client_config in benchmark_config["clientconfigs"]:
+            if "run_image" in client_config:
+                images.append(client_config["run_image"])
+            else:
+                images.append(None)
+        return images
+    elif "clientconfig" in benchmark_config:
+        # Single client config - return list with one image for consistency
+        if "run_image" in benchmark_config["clientconfig"]:
+            return [benchmark_config["clientconfig"]["run_image"]]
+        else:
+            return [None]
+    return []
+
+
 def extract_client_tool(benchmark_config, keyname="clientconfig"):
     client_tool = None
     if keyname in benchmark_config:
         if "tool" in benchmark_config[keyname]:
             client_tool = benchmark_config[keyname]["tool"]
     return client_tool
+
+
+def extract_client_tools(benchmark_config):
+    """Extract tools for both single and multiple client configs"""
+    if "clientconfigs" in benchmark_config:
+        # Multiple client configs - return list of tools
+        tools = []
+        for client_config in benchmark_config["clientconfigs"]:
+            if "tool" in client_config:
+                tools.append(client_config["tool"])
+            else:
+                tools.append(None)
+        return tools
+    elif "clientconfig" in benchmark_config:
+        # Single client config - return list with one tool for consistency
+        if "tool" in benchmark_config["clientconfig"]:
+            return [benchmark_config["clientconfig"]["tool"]]
+        else:
+            return [None]
+    return []
+
+
+def extract_client_configs(benchmark_config):
+    """Extract client configurations as a list for both single and multiple formats"""
+    if "clientconfigs" in benchmark_config:
+        # Multiple client configs
+        return benchmark_config["clientconfigs"]
+    elif "clientconfig" in benchmark_config:
+        # Single client config - return as list for consistency
+        return [benchmark_config["clientconfig"]]
+    return []

redis_benchmarks_specification/__runner__/args.py

@@ -208,6 +208,12 @@ def create_client_runner_args(project_name):
         type=int,
         help="override memtier number of runs for each benchmark. By default will run once each test",
     )
+    parser.add_argument(
+        "--container-timeout-buffer",
+        default=60,
+        type=int,
+        help="Buffer time in seconds to add to test-time for container timeout. Default is 60 seconds.",
+    )
     parser.add_argument(
         "--cluster-mode",
         default=False,