diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 1ec77821..ea0cdb61 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -21,6 +21,7 @@
 /src/aiconfigurator/sdk/perf_database.py @Arsene12358 @YijiaZhao @ilyasher @xutizhou @AichenF @tianhaox @jasonqinzhou
 /src/aiconfigurator/sdk/task.py @tianhaox @jasonqinzhou @ilyasher @Arsene12358
 /src/aiconfigurator/sdk/utils.py @tianhaox @jasonqinzhou @simone-chen @Arsene12358
+/src/aiconfigurator/sdk/suppport_matrix.py @Harrilee
 
 # cli
 /src/aiconfigurator/cli @Ethan-ES @tianhaox @jasonqinzhou @Arsene12358
@@ -71,6 +72,7 @@
 /tools/automation @tianhaox @Ethan-ES @jasonqinzhou @Arsene12358
 /tools/sanity_check @tianhaox @YijiaZhao @jasonqinzhou @Arsene12358
 /tools/simple_sdk_demo @tianhaox @jasonqinzhou @Arsene12358
+/tools/support_matrix/generate_support_matrix.py @Harrilee
 
 # misc
 /ATTRIBUTIONS.md @saturley-hall
@@ -85,4 +87,4 @@
 /.pre-commit-config.yaml @saturley-hall
 
 # CI/CD and workflows
-/.github/workflows @saturley-hall @tianhaox @jasonqinzhou @Arsene12358
\ No newline at end of file
+/.github/workflows @saturley-hall @tianhaox @jasonqinzhou @Arsene12358 @Harrilee
diff --git a/.github/workflows/daily-support-matrix.yml b/.github/workflows/daily-support-matrix.yml
new file mode 100644
index 00000000..cbcefcec
--- /dev/null
+++ b/.github/workflows/daily-support-matrix.yml
@@ -0,0 +1,30 @@
+name: "Daily Support Matrix Test"
+
+on:
+  schedule:
+    # Run daily at 7 AM PT / 11 PM China / 3 PM UTC
+    - cron: "0 15 * * *"
+  workflow_dispatch: # Allow manual trigger (with no inputs)
+
+jobs:
+  test-support-matrix:
+    name: Test Support Matrix
+    runs-on: ubuntu-latest
+    timeout-minutes: 480 # 8 hours
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Git LFS Pull
+        run: git lfs pull
+
+      - name: Build test container
+        run: |
+          docker build -f docker/Dockerfile -t aiconfigurator:test --target test .
+
+      - name: Run support matrix tests in container
+        run: |
+          docker run --name aic-support-matrix --env TEST_SUPPORT_MATRIX=true aiconfigurator:test \
+            pytest tests/sdk/support_matrix/test_support_matrix.py -v --tb=short
diff --git a/README.md b/README.md
index 702f8042..e71c2ee5 100644
--- a/README.md
+++ b/README.md
@@ -305,7 +305,11 @@ To go through the process, refer to the [guidance](collector/README.md) under th
 | gb200_sxm | TRTLLM(1.0.0rc6) | ✅ |
 | a100_sxm | TRTLLM(1.0.0) | ✅ |
 
-> **Note**: b200 and gb200 are under dev. Results are to be aligned. For preview now. 
+> **Note**: b200 and gb200 are under dev. Results are to be aligned. For preview now.
+
+#### Detailed Support Matrix
+
+For a comprehensive breakdown of which model/system/backend/version combinations are supported in both aggregated and disaggregated modes, refer to the [**support matrix CSV**](src/aiconfigurator/systems/support_matrix.csv). This file is automatically generated and tested to ensure accuracy across all supported configurations.
 
 ## Contributing and Development
 
diff --git a/docker/Dockerfile b/docker/Dockerfile
index cfae880a..b1fd8d28 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -33,3 +33,4 @@ RUN WHL=$(ls -d /wheelhouse/*) && \
 COPY pytest.ini /workspace/
 COPY tests/ /workspace/tests/
 COPY src/ /workspace/src/
+COPY tools/ /workspace/tools/
diff --git a/pyproject.toml b/pyproject.toml
index 4024688c..691ad358 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -54,6 +54,7 @@ dependencies = [
     "pydantic~=2.11.4",
     "pyyaml>=6.0",
     "scipy>=1.13.1",
+    "tqdm>=4.0.0",
     "uvicorn>=0.34.2",
     "bokeh",
     "nvidia-ml-py",
diff --git a/src/aiconfigurator/sdk/inference_session.py b/src/aiconfigurator/sdk/inference_session.py
index 615f7c40..b9ee730e 100644
--- a/src/aiconfigurator/sdk/inference_session.py
+++ b/src/aiconfigurator/sdk/inference_session.py
@@ -431,6 +431,7 @@ def _get_summary_df(
             Get all worker candidates based on give search space
             """
             summary_df = pd.DataFrame(columns=common.ColumnsStatic)
+            exceptions = []
 
             for parallel_config in parallel_config_list:
                 tp_size, pp_size, dp_size, moe_tp_size, moe_ep_size = parallel_config
@@ -480,13 +481,18 @@ def _get_summary_df(
                             )
                         else:  # larger b will always OOM
                             break
-                except Exception:
+                except Exception as e:
                     logger.exception(
                         f"Error getting candidate workers with parallel config: "
                         f"tp={tp_size}, pp={pp_size}, dp={dp_size}, moe_tp={moe_tp_size}, "
                         f"moe_ep={moe_ep_size}; skipping this combination"
                     )
+                    exceptions.append(e)
                     continue
+            if summary_df.empty:
+                raise RuntimeError(
+                    f"No results found for any parallel configuration. Showing last exception: {exceptions[-1]}"
+                ) from exceptions[-1]
             return summary_df
 
         def _find_best_result_under_constraints(
diff --git a/src/aiconfigurator/sdk/pareto_analysis.py b/src/aiconfigurator/sdk/pareto_analysis.py
index 7a9f141c..d9275a7d 100755
--- a/src/aiconfigurator/sdk/pareto_analysis.py
+++ b/src/aiconfigurator/sdk/pareto_analysis.py
@@ -113,6 +113,7 @@ def agg_pareto(
 
     # agg is agg server, the loop over parallel is outside here.
     results_df = pd.DataFrame(columns=ColumnsAgg)
+    exceptions = []
     for parallel_config in parallel_config_list:
         tp_size, pp_size, dp_size, moe_tp_size, moe_ep_size = parallel_config
         logger.debug(
@@ -151,8 +152,8 @@ def agg_pareto(
                     results_df = result_df
                 else:
                     results_df = pd.concat([results_df, result_df], axis=0, ignore_index=True)
-        except Exception:
-            logger.exception(
+        except Exception as e:
+            logger.info(
                 "Error getting candidate workers with parallel config: tp=%s, pp=%s, dp=%s, "
                 "moe_tp=%s, moe_ep=%s, skip this combination",
                 tp_size,
@@ -161,8 +162,15 @@ def agg_pareto(
                 moe_tp_size,
                 moe_ep_size,
             )
+            exceptions.append(e)
             continue
 
+    # If no results found, raise the last exception
+    if results_df.empty:
+        raise RuntimeError(
+            f"No results found for any parallel configuration. Showing last exception: {exceptions[-1]}"
+        ) from exceptions[-1]
+
     results_df = results_df.sort_values(by="tokens/s/gpu", ascending=False).reset_index(drop=True)
 
     return results_df
diff --git a/src/aiconfigurator/sdk/task.py b/src/aiconfigurator/sdk/task.py
index 5b0f63d8..a35789ba 100644
--- a/src/aiconfigurator/sdk/task.py
+++ b/src/aiconfigurator/sdk/task.py
@@ -1188,6 +1188,7 @@ def run(self, task_config: TaskConfig) -> dict[str, pd.DataFrame | None]:
                 serving_mode,
             )
             result = None
+            raise
 
         if result is None:
             logger.warning("No result found for %s in %s mode.", task_config.task_name, serving_mode)
diff --git a/src/aiconfigurator/systems/support_matrix.csv b/src/aiconfigurator/systems/support_matrix.csv
new file mode 100644
index 00000000..95b57527
--- /dev/null
+++ b/src/aiconfigurator/systems/support_matrix.csv
@@ -0,0 +1,599 @@
+Model,System,Backend,Version,Mode,Status,ErrMsg
+DEEPSEEK_V3,a100_sxm,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+DEEPSEEK_V3,a100_sxm,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+DEEPSEEK_V3,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+DEEPSEEK_V3,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+DEEPSEEK_V3,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+DEEPSEEK_V3,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+DEEPSEEK_V3,h100_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 363, in query\n    assert self._attention_tp_size == 1 or self._attention_dp_size == 1, (\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: We don't enable the path for SGLang to support TP>1 and DP>1 for attn simultaneously\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: We don't enable the path for SGLang to support TP>1 and DP>1 for attn simultaneously\n"
+DEEPSEEK_V3,h100_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 363, in query\n    assert self._attention_tp_size == 1 or self._attention_dp_size == 1, (\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: We don't enable the path for SGLang to support TP>1 and DP>1 for attn simultaneously\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: We don't enable the path for SGLang to support TP>1 and DP>1 for attn simultaneously\n"
+DEEPSEEK_V3,h100_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+DEEPSEEK_V3,h100_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 494, in _get_summary_df\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+DEEPSEEK_V3,h100_sxm,trtllm,1.0.0rc3,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+DEEPSEEK_V3,h100_sxm,trtllm,1.0.0rc3,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 494, in _get_summary_df\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+DEEPSEEK_V3,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+DEEPSEEK_V3,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+DEEPSEEK_V3,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+DEEPSEEK_V3,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+DEEPSEEK_V3,h200_sxm,trtllm,0.20.0,agg,PASS,
+DEEPSEEK_V3,h200_sxm,trtllm,0.20.0,disagg,PASS,
+DEEPSEEK_V3,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+DEEPSEEK_V3,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+DEEPSEEK_V3,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+DEEPSEEK_V3,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+DEEPSEEK_V3,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+DEEPSEEK_V3,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+DEEPSEEK_V3,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+DEEPSEEK_V3,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,a100_sxm,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,a100_sxm,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,b200_sxm,trtllm,1.0.0rc6,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,b200_sxm,trtllm,1.0.0rc6,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,gb200_sxm,trtllm,1.0.0rc6,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,gb200_sxm,trtllm,1.0.0rc6,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,trtllm,1.0.0rc3,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,trtllm,1.0.0rc3,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_120B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_120B,h200_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,trtllm,1.0.0rc3,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,trtllm,1.0.0rc3,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,trtllm,1.2.0rc2,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,trtllm,1.2.0rc2,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_120B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_120B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_120B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,a100_sxm,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,a100_sxm,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,b200_sxm,trtllm,1.0.0rc6,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,b200_sxm,trtllm,1.0.0rc6,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,gb200_sxm,trtllm,1.0.0rc6,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,gb200_sxm,trtllm,1.0.0rc6,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,trtllm,1.0.0rc3,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,trtllm,1.0.0rc3,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_20B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_20B,h200_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,trtllm,1.0.0rc3,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,trtllm,1.0.0rc3,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,trtllm,1.2.0rc2,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,trtllm,1.2.0rc2,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_20B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+GPT_OSS_20B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+GPT_OSS_20B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 502, in query\n    database.query_context_attention(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2053, in query_context_attention\n    latency = self._interp_3d(n, full_s, b, attention_dict, ""cubic"") * prefix_correction\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1864, in _interp_3d\n    return self._interp_2d_1d(x, y, z, data, method)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1890, in _interp_2d_1d\n    x_left, x_right = self._nearest_1d_point_helper(x, list(data.keys()))\n                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+LLAMA2_13B,a100_sxm,trtllm,1.0.0,agg,PASS,
+LLAMA2_13B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+LLAMA2_13B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA2_13B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA2_13B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA2_13B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA2_13B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA2_13B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA2_13B,h100_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA2_13B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA2_13B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA2_13B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA2_13B,h100_sxm,vllm,0.11.0,agg,PASS,
+LLAMA2_13B,h100_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA2_13B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA2_13B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA2_13B,h200_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA2_13B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA2_13B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA2_13B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA2_13B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+LLAMA2_13B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+LLAMA2_13B,h200_sxm,vllm,0.11.0,agg,PASS,
+LLAMA2_13B,h200_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA2_13B,l40s,trtllm,1.0.0,agg,PASS,
+LLAMA2_13B,l40s,trtllm,1.0.0,disagg,PASS,
+LLAMA2_70B,a100_sxm,trtllm,1.0.0,agg,PASS,
+LLAMA2_70B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+LLAMA2_70B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA2_70B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA2_70B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA2_70B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA2_70B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA2_70B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA2_70B,h100_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA2_70B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA2_70B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA2_70B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA2_70B,h100_sxm,vllm,0.11.0,agg,PASS,
+LLAMA2_70B,h100_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA2_70B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA2_70B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA2_70B,h200_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA2_70B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA2_70B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA2_70B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA2_70B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+LLAMA2_70B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+LLAMA2_70B,h200_sxm,vllm,0.11.0,agg,PASS,
+LLAMA2_70B,h200_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA2_70B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+LLAMA2_70B,l40s,trtllm,1.0.0,disagg,PASS,
+LLAMA2_7B,a100_sxm,trtllm,1.0.0,agg,PASS,
+LLAMA2_7B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+LLAMA2_7B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA2_7B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA2_7B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA2_7B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA2_7B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA2_7B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA2_7B,h100_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA2_7B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA2_7B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA2_7B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA2_7B,h100_sxm,vllm,0.11.0,agg,PASS,
+LLAMA2_7B,h100_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA2_7B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA2_7B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA2_7B,h200_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA2_7B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA2_7B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA2_7B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA2_7B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+LLAMA2_7B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+LLAMA2_7B,h200_sxm,vllm,0.11.0,agg,PASS,
+LLAMA2_7B,h200_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA2_7B,l40s,trtllm,1.0.0,agg,PASS,
+LLAMA2_7B,l40s,trtllm,1.0.0,disagg,PASS,
+LLAMA3.1_405B,a100_sxm,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+LLAMA3.1_405B,a100_sxm,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 494, in _get_summary_df\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+LLAMA3.1_405B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA3.1_405B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA3.1_405B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA3.1_405B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA3.1_405B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA3.1_405B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA3.1_405B,h100_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA3.1_405B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA3.1_405B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA3.1_405B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA3.1_405B,h100_sxm,vllm,0.11.0,agg,PASS,
+LLAMA3.1_405B,h100_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA3.1_405B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA3.1_405B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA3.1_405B,h200_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA3.1_405B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA3.1_405B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA3.1_405B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA3.1_405B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+LLAMA3.1_405B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+LLAMA3.1_405B,h200_sxm,vllm,0.11.0,agg,PASS,
+LLAMA3.1_405B,h200_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA3.1_405B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+LLAMA3.1_405B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 494, in _get_summary_df\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+LLAMA3.1_70B,a100_sxm,trtllm,1.0.0,agg,PASS,
+LLAMA3.1_70B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+LLAMA3.1_70B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA3.1_70B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA3.1_70B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA3.1_70B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA3.1_70B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA3.1_70B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA3.1_70B,h100_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA3.1_70B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA3.1_70B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA3.1_70B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA3.1_70B,h100_sxm,vllm,0.11.0,agg,PASS,
+LLAMA3.1_70B,h100_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA3.1_70B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA3.1_70B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA3.1_70B,h200_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA3.1_70B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA3.1_70B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA3.1_70B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA3.1_70B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+LLAMA3.1_70B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+LLAMA3.1_70B,h200_sxm,vllm,0.11.0,agg,PASS,
+LLAMA3.1_70B,h200_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA3.1_70B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+LLAMA3.1_70B,l40s,trtllm,1.0.0,disagg,PASS,
+LLAMA3.1_8B,a100_sxm,trtllm,1.0.0,agg,PASS,
+LLAMA3.1_8B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+LLAMA3.1_8B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA3.1_8B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA3.1_8B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+LLAMA3.1_8B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+LLAMA3.1_8B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA3.1_8B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA3.1_8B,h100_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA3.1_8B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA3.1_8B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA3.1_8B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA3.1_8B,h100_sxm,vllm,0.11.0,agg,PASS,
+LLAMA3.1_8B,h100_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA3.1_8B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+LLAMA3.1_8B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+LLAMA3.1_8B,h200_sxm,trtllm,0.20.0,agg,PASS,
+LLAMA3.1_8B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+LLAMA3.1_8B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+LLAMA3.1_8B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+LLAMA3.1_8B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+LLAMA3.1_8B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+LLAMA3.1_8B,h200_sxm,vllm,0.11.0,agg,PASS,
+LLAMA3.1_8B,h200_sxm,vllm,0.11.0,disagg,PASS,
+LLAMA3.1_8B,l40s,trtllm,1.0.0,agg,PASS,
+LLAMA3.1_8B,l40s,trtllm,1.0.0,disagg,PASS,
+MOE_Mixtral8x22B,a100_sxm,trtllm,1.0.0,agg,PASS,
+MOE_Mixtral8x22B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+MOE_Mixtral8x22B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+MOE_Mixtral8x22B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+MOE_Mixtral8x22B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+MOE_Mixtral8x22B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+MOE_Mixtral8x22B,h100_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x22B,h100_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x22B,h100_sxm,trtllm,0.20.0,agg,PASS,
+MOE_Mixtral8x22B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+MOE_Mixtral8x22B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+MOE_Mixtral8x22B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+MOE_Mixtral8x22B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x22B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x22B,h200_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x22B,h200_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x22B,h200_sxm,trtllm,0.20.0,agg,PASS,
+MOE_Mixtral8x22B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+MOE_Mixtral8x22B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+MOE_Mixtral8x22B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+MOE_Mixtral8x22B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+MOE_Mixtral8x22B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+MOE_Mixtral8x22B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x22B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x22B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x22B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x7B,a100_sxm,trtllm,1.0.0,agg,PASS,
+MOE_Mixtral8x7B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+MOE_Mixtral8x7B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+MOE_Mixtral8x7B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+MOE_Mixtral8x7B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+MOE_Mixtral8x7B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+MOE_Mixtral8x7B,h100_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x7B,h100_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x7B,h100_sxm,trtllm,0.20.0,agg,PASS,
+MOE_Mixtral8x7B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+MOE_Mixtral8x7B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+MOE_Mixtral8x7B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+MOE_Mixtral8x7B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x7B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x7B,h200_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x7B,h200_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x7B,h200_sxm,trtllm,0.20.0,agg,PASS,
+MOE_Mixtral8x7B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+MOE_Mixtral8x7B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+MOE_Mixtral8x7B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+MOE_Mixtral8x7B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+MOE_Mixtral8x7B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+MOE_Mixtral8x7B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x7B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+MOE_Mixtral8x7B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+MOE_Mixtral8x7B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+Nemotron_super_v1.1,a100_sxm,trtllm,1.0.0,agg,PASS,
+Nemotron_super_v1.1,a100_sxm,trtllm,1.0.0,disagg,PASS,
+Nemotron_super_v1.1,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+Nemotron_super_v1.1,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+Nemotron_super_v1.1,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+Nemotron_super_v1.1,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+Nemotron_super_v1.1,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+Nemotron_super_v1.1,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+Nemotron_super_v1.1,h100_sxm,trtllm,0.20.0,agg,PASS,
+Nemotron_super_v1.1,h100_sxm,trtllm,0.20.0,disagg,PASS,
+Nemotron_super_v1.1,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+Nemotron_super_v1.1,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+Nemotron_super_v1.1,h100_sxm,vllm,0.11.0,agg,PASS,
+Nemotron_super_v1.1,h100_sxm,vllm,0.11.0,disagg,PASS,
+Nemotron_super_v1.1,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+Nemotron_super_v1.1,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+Nemotron_super_v1.1,h200_sxm,trtllm,0.20.0,agg,PASS,
+Nemotron_super_v1.1,h200_sxm,trtllm,0.20.0,disagg,PASS,
+Nemotron_super_v1.1,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+Nemotron_super_v1.1,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+Nemotron_super_v1.1,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+Nemotron_super_v1.1,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+Nemotron_super_v1.1,h200_sxm,vllm,0.11.0,agg,PASS,
+Nemotron_super_v1.1,h200_sxm,vllm,0.11.0,disagg,PASS,
+Nemotron_super_v1.1,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+Nemotron_super_v1.1,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_1.5B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN2.5_1.5B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_1.5B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_1.5B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_1.5B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_1.5B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_1.5B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_1.5B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_1.5B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_1.5B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_1.5B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_1.5B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_1.5B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_1.5B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_1.5B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_1.5B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_1.5B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_1.5B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_1.5B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_1.5B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_1.5B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN2.5_1.5B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN2.5_1.5B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_1.5B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_1.5B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN2.5_1.5B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_32B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN2.5_32B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_32B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_32B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_32B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_32B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_32B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_32B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_32B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_32B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_32B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_32B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_32B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_32B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_32B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_32B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_32B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_32B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_32B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_32B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_32B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN2.5_32B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN2.5_32B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_32B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_32B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN2.5_32B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_72B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN2.5_72B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_72B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_72B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_72B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_72B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_72B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_72B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_72B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_72B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_72B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_72B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_72B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_72B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_72B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_72B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_72B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_72B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_72B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_72B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_72B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN2.5_72B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN2.5_72B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_72B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_72B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 171, in agg_pareto\n    f""No results found for any parallel configuration. Showing last exception: {exceptions[-1]}""\n                                                                                ~~~~~~~~~~^^^^\nIndexError: list index out of range\n"
+QWEN2.5_72B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_7B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN2.5_7B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN2.5_7B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_7B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_7B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN2.5_7B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN2.5_7B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_7B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_7B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_7B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_7B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_7B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_7B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_7B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_7B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN2.5_7B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN2.5_7B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN2.5_7B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN2.5_7B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN2.5_7B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN2.5_7B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN2.5_7B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN2.5_7B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN2.5_7B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN2.5_7B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN2.5_7B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN3_0.6B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN3_0.6B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN3_0.6B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_0.6B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_0.6B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_0.6B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_0.6B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_0.6B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_0.6B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_0.6B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_0.6B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_0.6B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_0.6B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_0.6B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_0.6B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_0.6B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_0.6B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_0.6B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_0.6B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_0.6B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_0.6B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_0.6B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_0.6B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_0.6B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_0.6B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN3_0.6B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN3_1.7B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN3_1.7B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN3_1.7B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_1.7B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_1.7B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_1.7B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_1.7B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_1.7B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_1.7B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_1.7B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_1.7B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_1.7B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_1.7B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_1.7B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_1.7B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_1.7B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_1.7B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_1.7B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_1.7B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_1.7B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_1.7B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_1.7B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_1.7B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_1.7B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_1.7B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN3_1.7B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN3_235B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN3_235B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN3_235B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_235B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_235B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_235B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_235B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_235B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_235B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_235B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_235B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_235B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_235B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_235B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_235B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_235B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_235B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_235B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_235B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_235B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_235B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_235B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_235B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_235B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_235B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_235B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,a100_sxm,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,a100_sxm,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,b200_sxm,trtllm,1.0.0rc6,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,b200_sxm,trtllm,1.0.0rc6,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,gb200_sxm,trtllm,1.0.0rc6,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,gb200_sxm,trtllm,1.0.0rc6,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,sglang,0.5.1.post1,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 45, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 352, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 157, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/sglang_backend.py"", line 93, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,sglang,0.5.1.post1,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2651, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,trtllm,1.0.0rc3,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,trtllm,1.0.0rc3,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_30B_A3B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_30B_A3B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_30B_A3B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_30B_A3B,h200_sxm,trtllm,0.20.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h200_sxm,trtllm,0.20.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_30B_A3B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_30B_A3B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_30B_A3B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_30B_A3B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_30B_A3B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_30B_A3B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_30B_A3B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_32B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN3_32B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN3_32B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_32B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_32B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_32B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_32B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_32B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_32B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_32B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_32B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_32B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_32B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_32B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_32B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_32B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_32B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_32B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_32B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_32B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_32B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_32B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_32B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_32B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_32B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN3_32B,l40s,trtllm,1.0.0,disagg,PASS,
+QWEN3_480B,a100_sxm,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_480B,a100_sxm,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_480B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_480B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_480B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_480B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_480B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_480B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_480B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_480B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_480B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_480B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_480B,h100_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_480B,h100_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_480B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_480B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_480B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_480B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_480B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_480B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_480B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_480B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_480B,h200_sxm,vllm,0.11.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 241, in _agg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_480B,h200_sxm,vllm,0.11.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 113, in run_single_test\n    task_config = TaskConfig(**task_config_kwargs)\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 762, in __init__\n    self.config, applied_layers = TaskConfigFactory.create(ctx)\n                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 118, in create\n    _deep_merge(config_dict, layer.resolve(ctx))\n                             ^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 43, in resolve\n    payload = self.data(ctx) if callable(self.data) else self.data\n              ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 354, in _disagg_defaults_layer\n    raise NotImplementedError(""MoE is not implemented for vllm backend"")\nNotImplementedError: MoE is not implemented for vllm backend\n"
+QWEN3_480B,l40s,trtllm,1.0.0,agg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 47, in run_agg\n    summary = self._agg_cache[isl][osl][b][ctx_tokens]\n              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^\nKeyError: 4000\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 141, in agg_pareto\n    summary = sess.find_best_agg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 98, in find_best_agg_result_under_constraints\n    return self._backend.find_best_agg_result_under_constraints(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 354, in find_best_agg_result_under_constraints\n    summary = self.run_agg(\n              ^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 159, in run_agg\n    mix_step_latency = _get_mix_step_latency(\n                       ^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/trtllm_backend.py"", line 95, in _get_mix_step_latency\n    summary = self.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1179, in run\n    result = self.run_agg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1006, in run_agg\n    result_df = pa.agg_pareto(\n                ^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 170, in agg_pareto\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_480B,l40s,trtllm,1.0.0,disagg,FAIL,"Traceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 471, in _get_summary_df\n    summary = sess.run_static(\n              ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 65, in run_static\n    return self._backend.run_static(\n           ^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 115, in run_static\n    context_latency_dict = _run_context(batch_size, isl, prefix)\n                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/backends/base_backend.py"", line 73, in _run_context\n    latency = op.query(database, x=x, batch_size=batch_size, beam_width=1, s=isl, prefix=prefix)\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/operations.py"", line 181, in query\n    database.query_moe(\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 2690, in query_moe\n    num_left, num_right = self._nearest_1d_point_helper(num_tokens, list(moe_dict.keys()), inner_only=False)\n                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/perf_database.py"", line 1787, in _nearest_1d_point_helper\n    assert values is not None and len(values) >= 2, ""values is None or len(values) < 2""\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nAssertionError: values is None or len(values) < 2\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File ""/home/harrli/Projects/aiconfigurator/tests/sdk/support_matrix/suppport_matrix.py"", line 117, in run_single_test\n    result = runner.run(task_config)\n             ^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1181, in run\n    result = self.run_disagg(task_config.config)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/task.py"", line 1142, in run_disagg\n    result_df = pa.disagg_pareto(\n                ^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/pareto_analysis.py"", line 278, in disagg_pareto\n    summary = disagg_sess.find_best_disagg_result_under_constraints(\n              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 631, in find_best_disagg_result_under_constraints\n    prefill_summary_df = _get_summary_df(\n                         ^^^^^^^^^^^^^^^^\n  File ""/home/harrli/Projects/aiconfigurator/src/aiconfigurator/sdk/inference_session.py"", line 493, in _get_summary_df\n    raise RuntimeError(\nRuntimeError: No results found for any parallel configuration. Showing last exception: values is None or len(values) < 2\n"
+QWEN3_8B,a100_sxm,trtllm,1.0.0,agg,PASS,
+QWEN3_8B,a100_sxm,trtllm,1.0.0,disagg,PASS,
+QWEN3_8B,b200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_8B,b200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_8B,gb200_sxm,trtllm,1.0.0rc6,agg,PASS,
+QWEN3_8B,gb200_sxm,trtllm,1.0.0rc6,disagg,PASS,
+QWEN3_8B,h100_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_8B,h100_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_8B,h100_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_8B,h100_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_8B,h100_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_8B,h100_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_8B,h100_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_8B,h100_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_8B,h200_sxm,sglang,0.5.1.post1,agg,PASS,
+QWEN3_8B,h200_sxm,sglang,0.5.1.post1,disagg,PASS,
+QWEN3_8B,h200_sxm,trtllm,0.20.0,agg,PASS,
+QWEN3_8B,h200_sxm,trtllm,0.20.0,disagg,PASS,
+QWEN3_8B,h200_sxm,trtllm,1.0.0rc3,agg,PASS,
+QWEN3_8B,h200_sxm,trtllm,1.0.0rc3,disagg,PASS,
+QWEN3_8B,h200_sxm,trtllm,1.2.0rc2,agg,PASS,
+QWEN3_8B,h200_sxm,trtllm,1.2.0rc2,disagg,PASS,
+QWEN3_8B,h200_sxm,vllm,0.11.0,agg,PASS,
+QWEN3_8B,h200_sxm,vllm,0.11.0,disagg,PASS,
+QWEN3_8B,l40s,trtllm,1.0.0,agg,PASS,
+QWEN3_8B,l40s,trtllm,1.0.0,disagg,PASS,
diff --git a/tests/sdk/support_matrix/suppport_matrix.py b/tests/sdk/support_matrix/suppport_matrix.py
new file mode 100644
index 00000000..220dab05
--- /dev/null
+++ b/tests/sdk/support_matrix/suppport_matrix.py
@@ -0,0 +1,257 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import logging
+import traceback
+
+from packaging.version import Version
+from tqdm import tqdm
+
+from aiconfigurator.sdk import common, perf_database
+from aiconfigurator.sdk.task import TaskConfig, TaskRunner
+
+logger = logging.getLogger(__name__)
+
+# Test configuration constants
+TOTAL_GPUS = 128
+ISL = 4000
+OSL = 500
+PREFIX = 500
+TTFT = 2000.0
+TPOT = 50.0
+
+
+class SupportMatrix:
+    def __init__(self):
+        self.models: set[str] = self.get_models()
+        # database structure: {system: {backend: {version}}}
+        self.databases: dict[str, dict[str, dict[str, str]]] = self.load_databases()
+
+    def get_models(self):
+        return set[str](common.SupportedModels.keys())
+
+    def get_systems(self):
+        return set(common.SupportedSystems)
+
+    def get_backends(self):
+        return set(x.value for x in common.BackendName)
+
+    def load_databases(self):
+        return perf_database.get_all_databases()
+
+    def __get_hardware_and_backend_combinations(self) -> list[tuple[str, str, str]]:
+        """
+        Iterate over all combinations of hardware, and inference backend, version.
+        """
+        for hardware in self.get_systems():
+            for backend in self.get_backends():
+                for version in self.databases[hardware][backend]:
+                    yield hardware, backend, version
+
+    def __get_model_and_hardware_and_backend_combinations(self) -> list[tuple[str, str, str, str]]:
+        """
+        Iterate over all combinations of models, hardware, and inference backend, version.
+        """
+        for hardware, backend, version in self.__get_hardware_and_backend_combinations():
+            for model in self.models:
+                yield model, hardware, backend, version
+
+    def generate_combinations(self):
+        """
+        Generate all combinations of models, hardware, and inference backend, version.
+        """
+        # get all combinations of hardware, and inference backend, version
+        combinations = list(self.__get_model_and_hardware_and_backend_combinations())
+        return combinations
+
+    def run_single_test(
+        self,
+        model: str,
+        system: str,
+        backend: str,
+        version: str,
+    ) -> tuple[dict[str, bool], dict[str, str | None]]:
+        """
+        Run a single configuration test for both agg and disagg modes.
+
+        Args:
+            model: Model name
+            system: System/hardware name
+            backend: Backend name
+            version: Backend version
+
+        Returns:
+            Tuple of (dict with results, dict with error messages)
+            Both dicts have keys "agg" and "disagg"
+        """
+        modes_to_test = ["agg", "disagg"]
+        results = {}
+        error_messages = {}
+
+        for mode in modes_to_test:
+            try:
+                # Create TaskConfig for the test
+                task_config_kwargs = {
+                    "serving_mode": mode,
+                    "model_name": model,
+                    "system_name": system,
+                    "backend_name": backend,
+                    "backend_version": version,
+                    "total_gpus": TOTAL_GPUS,
+                    "isl": ISL,
+                    "osl": OSL,
+                    "prefix": PREFIX,
+                    "ttft": TTFT,
+                    "tpot": TPOT,
+                }
+
+                # For disagg mode, set decode_system_name
+                if mode == "disagg":
+                    task_config_kwargs["decode_system_name"] = system
+
+                task_config = TaskConfig(**task_config_kwargs)
+
+                # Run the configuration
+                runner = TaskRunner()
+                result = runner.run(task_config)
+
+                # Check if we got valid results
+                pareto_frontier_df = result.get("pareto_frontier_df")
+                if pareto_frontier_df is not None and not pareto_frontier_df.empty:
+                    results[mode] = True
+                    error_messages[mode] = None
+                else:  # pragma: no cover
+                    logger.debug(
+                        "Configuration returned no results: %s, %s, %s, %s, mode=%s",
+                        model,
+                        system,
+                        backend,
+                        version,
+                        mode,
+                    )
+                    results[mode] = False
+                    error_messages[mode] = "Configuration returned no results, failed to catch traceback"
+
+            except Exception as e:
+                logger.debug(
+                    "Configuration failed: %s, %s, %s, %s, mode=%s - Error: %s",
+                    model,
+                    system,
+                    backend,
+                    version,
+                    mode,
+                    str(e),
+                )
+                results[mode] = False
+                error_messages[mode] = traceback.format_exc()
+            finally:
+                # format error messages to one line with "\n" as separator
+                error_messages[mode] = error_messages[mode].replace("\n", "\\n") if error_messages[mode] else None
+        return results, error_messages
+
+    def test_support_matrix(self) -> list[tuple[str, str, str, str, str, bool, str | None]]:
+        """
+        Test whether each combination is supported by AIC.
+        Tests both agg and disagg modes for each combination and captures error messages.
+
+        Returns:
+            List of tuples (model, system, backend, version, mode, success, err_msg)
+            Returns separate entries for agg and disagg modes
+        """
+        # Print configuration
+        print("\n" + "=" * 80)
+        print("AIConfigurator Support Matrix Test")
+        print("=" * 80)
+        print("Testing both agg and disagg modes for all combinations")
+        print(f"Total GPUs: {TOTAL_GPUS}")
+        print(f"Input Sequence Length (ISL): {ISL}")
+        print(f"Output Sequence Length (OSL): {OSL}")
+        print(f"Prefix: {PREFIX}")
+        print(f"Target TTFT: {TTFT}ms")
+        print(f"Target TPOT: {TPOT}ms")
+        print("=" * 80 + "\n")
+
+        combinations = self.generate_combinations()
+        results = []
+
+        # Use tqdm for progress tracking
+        for model, system, backend, version in tqdm(
+            combinations,
+            desc="Testing support matrix",
+            unit="config",
+        ):
+            success_dict, error_dict = self.run_single_test(
+                model=model,
+                system=system,
+                backend=backend,
+                version=version,
+            )
+
+            # Add separate entries for agg and disagg modes
+            for mode in success_dict:
+                results.append((model, system, backend, version, mode, success_dict[mode], error_dict[mode]))
+
+        # Sort results by (model, system, backend, version, mode)
+        results.sort(key=lambda x: (x[0], x[1], x[2], Version(x[3]), x[4]))
+
+        # Print results summary
+        self._print_results_summary(results)
+
+        return results
+
+    def _print_results_summary(self, results: list[tuple[str, str, str, str, str, bool, str | None]]) -> None:
+        """Print summary of test results."""
+        total_tests = len(results)
+        passed = sum(1 for _, _, _, _, _, success, _ in results if success)
+        failed = total_tests - passed
+
+        print("\n" + "=" * 80)
+        print("Test Results Summary")
+        print("=" * 80)
+        print(f"Total configurations tested: {total_tests}")
+        print(f"✓ Passed: {passed} ({100 * passed / total_tests:.1f}%)")
+        print(f"✗ Failed: {failed} ({100 * failed / total_tests:.1f}%)")
+        print("=" * 80)
+
+        # Group results by status
+        passed_configs = []
+        failed_configs = []
+
+        for model, system, backend, version, mode, success, err_msg in results:
+            config = (model, system, backend, version, mode)
+            if success:
+                passed_configs.append(config)
+            else:
+                failed_configs.append(config)
+
+        # Print passed configurations
+        if passed_configs:
+            print(f"\n✓ Passed Configurations ({len(passed_configs)}):")
+            for model, system, backend, version, mode in sorted(passed_configs):
+                print(f"  • {model} on {system} with {backend} v{version} ({mode})")
+
+        # Print failed configurations
+        if failed_configs:
+            print(f"\n✗ Failed Configurations ({len(failed_configs)}):")
+            for model, system, backend, version, mode in sorted(failed_configs):
+                print(f"  • {model} on {system} with {backend} v{version} ({mode})")
+
+    def save_results_to_csv(
+        self, results: list[tuple[str, str, str, str, str, bool, str | None]], output_file: str
+    ) -> None:
+        """
+        Save test results to a CSV file.
+
+        Args:
+            results: List of tuples (model, system, backend, version, mode, success, err_msg)
+            output_file: Path to the output CSV file
+        """
+
+        with open(output_file, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(["Model", "System", "Backend", "Version", "Mode", "Status", "ErrMsg"])
+            for model, system, backend, version, mode, success, err_msg in results:
+                status = "PASS" if success else "FAIL"
+                writer.writerow([model, system, backend, version, mode, status, err_msg or ""])
+        print(f"\nResults saved to: {output_file}")
diff --git a/tests/sdk/support_matrix/test_support_matrix.py b/tests/sdk/support_matrix/test_support_matrix.py
new file mode 100644
index 00000000..378b166f
--- /dev/null
+++ b/tests/sdk/support_matrix/test_support_matrix.py
@@ -0,0 +1,288 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import os
+import subprocess
+import sys
+from collections import defaultdict
+
+import pytest
+from packaging.version import Version
+
+# Add tests directory to path for support_matrix module
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from sdk.support_matrix.suppport_matrix import SupportMatrix
+
+SUPPORT_MATRIX_CSV = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))),
+    "src",
+    "aiconfigurator",
+    "systems",
+    "support_matrix.csv",
+)
+
+GENERATE_SCRIPT = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))),
+    "tools",
+    "support_matrix",
+    "generate_support_matrix.py",
+)
+
+
+@pytest.fixture
+def csv_data():
+    """
+    Fixture that reads the support matrix CSV and returns header and data rows.
+
+    Returns:
+        tuple: (header, data_rows) where:
+            - header: list of column names
+            - data_rows: list of lists containing the data
+    """
+    assert os.path.exists(SUPPORT_MATRIX_CSV), f"Support matrix CSV not found at {SUPPORT_MATRIX_CSV}"
+
+    with open(SUPPORT_MATRIX_CSV, newline="") as f:
+        reader = csv.reader(f)
+        rows = list(reader)
+
+    assert len(rows) > 0, "CSV file is empty"
+
+    header = rows[0]
+    data_rows = rows[1:]
+
+    return header, data_rows
+
+
+@pytest.fixture(scope="session")
+def fresh_csv_data(tmp_path_factory):
+    """
+    Fixture that generates a fresh support matrix CSV and returns header and data rows.
+    Uses session scope to ensure the CSV is only generated once per test session (results are cached by pytest).
+
+    Returns:
+        tuple: (header, data_rows) where:
+            - header: list of column names
+            - data_rows: list of lists containing the data
+    """
+    # Create a temporary directory for this session
+    tmp_dir = tmp_path_factory.mktemp("support_matrix")
+    output_csv = tmp_dir / "support_matrix_fresh.csv"
+
+    # Run the generate script
+    result = subprocess.run(
+        ["python", GENERATE_SCRIPT, "--output", str(output_csv)],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    if result.returncode != 0:
+        pytest.fail(f"Failed to generate support matrix: {result.stderr}")
+
+    assert output_csv.exists(), f"Generated CSV not found at {output_csv}"
+
+    # Read the generated CSV
+    with open(output_csv, newline="") as f:
+        reader = csv.reader(f)
+        rows = list(reader)
+
+    assert len(rows) > 0, "Generated CSV file is empty"
+
+    header = rows[0]
+    data_rows = rows[1:]
+
+    return header, data_rows
+
+
+@pytest.mark.skipif(
+    os.environ.get("TEST_SUPPORT_MATRIX", "").lower() != "true",
+    reason="TEST_SUPPORT_MATRIX environment variable must be set to 'true' to run these tests",
+)
+class TestSupportMatrix:
+    """
+    Test suite for the support matrix.
+    Set TEST_SUPPORT_MATRIX=true to enable these tests.
+    """
+
+    def test_system_and_backend_matches_database(self):
+        """
+        Test that the system and backend defined in the support matrix matches the database.
+        """
+        support_matrix = SupportMatrix()
+        systems_in_database = set(support_matrix.databases.keys())
+        backends_in_database = {
+            backend for system in systems_in_database for backend in support_matrix.databases[system]
+        }
+        assert systems_in_database == support_matrix.get_systems()
+        assert backends_in_database == support_matrix.get_backends()
+
+    def test_csv_sanity_check(self, csv_data):
+        """
+        Test that the CSV file exists, has correct structure, and contains data.
+        """
+        header, data_rows = csv_data
+
+        # Check header row
+        expected_header = ["Model", "System", "Backend", "Version", "Mode", "Status", "ErrMsg"]
+        assert header == expected_header, f"Expected header {expected_header}, got {header}"
+
+        # Check that there are data rows
+        assert len(data_rows) > 0, "CSV file has header but no data rows"
+
+        # Validate data rows have correct number of columns
+        for i, row in enumerate(data_rows, start=2):
+            assert len(row) == len(expected_header), f"Row {i} has {len(row)} columns, expected {len(expected_header)}"
+
+            # Check that Mode column has valid values
+            mode = row[4]
+            assert mode in ["agg", "disagg"], f"Row {i}: Invalid mode '{mode}', expected 'agg' or 'disagg'"
+
+            # Check that Status column has valid values
+            status = row[5]
+            assert status in ["PASS", "FAIL"], f"Row {i}: Invalid status '{status}', expected 'PASS' or 'FAIL'"
+
+    def test_range_matches_database(self, csv_data):
+        """
+        Test that the CSV contains exactly the combinations expected from the database.
+        Each combination should appear twice (once for agg, once for disagg mode).
+        """
+        header, data_rows = csv_data
+
+        # Get expected combinations from the support matrix
+        support_matrix = SupportMatrix()
+        expected_base_combinations = set(support_matrix.generate_combinations())
+
+        # Each base combination should have both agg and disagg entries
+        expected_combinations = set()
+        for model, system, backend, version in expected_base_combinations:
+            expected_combinations.add((model, system, backend, version, "agg"))
+            expected_combinations.add((model, system, backend, version, "disagg"))
+
+        # Extract actual combinations from CSV (convert to dict format for easy access)
+        # Header indices: Model=0, System=1, Backend=2, Version=3, Mode=4
+        actual_combinations = {(row[0], row[1], row[2], row[3], row[4]) for row in data_rows}
+
+        # Compare sets
+        assert expected_combinations == actual_combinations, (
+            f"CSV combinations don't match expected combinations.\n"
+            f"Missing in CSV: {expected_combinations - actual_combinations}\n"
+            f"Extra in CSV: {actual_combinations - expected_combinations}"
+            + "\n\nIf these are intentional improvements, update the committed support_matrix.csv"
+            " with the `tools/support_matrix/generate_support_matrix.py` script and commit the changes."
+        )
+
+    def test_no_deprecated_support(self, csv_data, fresh_csv_data):
+        """
+        Test that all previously supported items are still supported.
+        This ensures no regression - configurations that used to work should still work.
+        """
+        _, old_data_rows = csv_data
+        _, new_data_rows = fresh_csv_data
+
+        # Build a dict of (model, system, backend, version, mode) -> status for old CSV
+        # Status indices: Model=0, System=1, Backend=2, Version=3, Mode=4, Status=5
+        old_status_map = {(row[0], row[1], row[2], row[3], row[4]): row[5] for row in old_data_rows}
+
+        # Build a dict for new CSV
+        new_status_map = {(row[0], row[1], row[2], row[3], row[4]): row[5] for row in new_data_rows}
+
+        # Find configurations that were PASS in old but are now FAIL in new
+        deprecated_support = []
+        for config, old_status in old_status_map.items():
+            if old_status == "PASS":
+                new_status = new_status_map.get(config)
+                if new_status == "FAIL":
+                    model, system, backend, version, mode = config
+                    deprecated_support.append(f"{model} on {system} with {backend} v{version} ({mode})")
+
+        # Assert no regressions
+        assert len(deprecated_support) == 0, (
+            f"Found {len(deprecated_support)} previously supported configurations that are now failing:\n"
+            + "\n".join(f"  - {item}" for item in deprecated_support)
+            + "\n\nIf these are intentional improvements, update the committed support_matrix.csv"
+            " with the `tools/support_matrix/generate_support_matrix.py` script and commit the changes."
+        )
+
+    def test_no_undocumented_support(self, csv_data, fresh_csv_data):
+        """
+        Test that all previously unsupported items are still unsupported.
+        This ensures no accidental new support - configurations that used to fail should still fail
+        unless there was an intentional change.
+        """
+        _, old_data_rows = csv_data
+        _, new_data_rows = fresh_csv_data
+
+        # Build a dict of (model, system, backend, version, mode) -> status for old CSV
+        old_status_map = {(row[0], row[1], row[2], row[3], row[4]): row[5] for row in old_data_rows}
+
+        # Build a dict for new CSV
+        new_status_map = {(row[0], row[1], row[2], row[3], row[4]): row[5] for row in new_data_rows}
+
+        # Find configurations that were FAIL in old but are now PASS in new
+        undocumented_support = []
+        for config, old_status in old_status_map.items():
+            if old_status == "FAIL":
+                new_status = new_status_map.get(config)
+                if new_status == "PASS":
+                    model, system, backend, version, mode = config
+                    undocumented_support.append(f"{model} on {system} with {backend} v{version} ({mode})")
+
+        # If there are newly supported configurations, list them
+        # This is not necessarily a failure - it might be intentional
+        # But we want to be explicit about it
+        if len(undocumented_support) > 0:
+            message = (
+                f"Found {len(undocumented_support)} newly supported configurations:\n"
+                + "\n".join(f"  - {item}" for item in undocumented_support)
+                + "\n\nIf these are intentional improvements, update the committed support_matrix.csv"
+                " with the `tools/support_matrix/generate_support_matrix.py` script and commit the changes."
+            )
+            pytest.fail(message)
+
+    def test_newer_versions_have_no_narrower_support(self, csv_data):
+        """
+        For each (model, system, backend), get its latest version and second latest version.
+        If the latest version is not supported, the second latest version should also not be supported.
+        """
+        _, data_rows = csv_data
+
+        # Group data by (model, system, backend, mode)
+        # Key: (model, system, backend, mode) -> List of (version, status)
+        grouped_data = defaultdict(list)
+
+        for row in data_rows:
+            model, system, backend, version, mode, status = row[0], row[1], row[2], row[3], row[4], row[5]
+            key = (model, system, backend, mode)
+            grouped_data[key].append((version, status))
+
+        violations = []
+
+        # For each group, check the version constraint
+        for (model, system, backend, mode), version_status_list in grouped_data.items():
+            # Sort by version (descending - latest first)
+            sorted_versions = sorted(version_status_list, key=lambda x: Version(x[0]), reverse=True)
+
+            # Need at least 2 versions to compare
+            if len(sorted_versions) < 2:
+                continue
+
+            latest_version, latest_status = sorted_versions[0]
+            second_latest_version, second_latest_status = sorted_versions[1]
+
+            # If latest is not supported (FAIL), second latest should also not be supported (FAIL)
+            # Using set subtraction: if latest is FAIL, we expect second_latest to be FAIL
+            # Violation: latest is FAIL but second_latest is PASS
+            if latest_status == "FAIL" and second_latest_status == "PASS":
+                violations.append(
+                    f"{model} on {system} with {backend} ({mode}): "
+                    f"Latest version {latest_version} is FAIL, but older version {second_latest_version} is PASS"
+                )
+
+        # Assert no violations found
+        assert len(violations) == 0, (
+            f"Found {len(violations)} cases where newer versions have narrower support than older versions:\n"
+            + "\n".join(f"  - {item}" for item in violations)
+            + "\n\nNewer versions should not have narrower support than older versions."
+        )
diff --git a/tools/support_matrix/generate_support_matrix.py b/tools/support_matrix/generate_support_matrix.py
new file mode 100644
index 00000000..d116fa13
--- /dev/null
+++ b/tools/support_matrix/generate_support_matrix.py
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Script to iterate over all model/system/backend/version combinations for complete support matrix generation
+
+Usage:
+    --output <output_file.csv> Save results to a CSV file
+"""
+
+import argparse
+import logging
+import os
+import sys
+
+# Add tests directory to path for support_matrix module
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "tests"))
+
+from sdk.support_matrix.suppport_matrix import SupportMatrix
+
+
+def main():
+    # Default output location: <package>/systems/support_matrix.csv
+    default_output = os.path.join(
+        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+        "src",
+        "aiconfigurator",
+        "systems",
+        "support_matrix.csv",
+    )
+
+    parser = argparse.ArgumentParser(
+        description="Test AIConfigurator support matrix across all model/system/backend combinations"
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default=default_output,
+        help=f"Output file to save results (CSV format) (default: {default_output})",
+    )
+
+    args = parser.parse_args()
+
+    print(f"Saving results to {args.output}")
+
+    # Setup logging
+    logging.basicConfig(
+        level=logging.CRITICAL,
+        format="%(levelname)s %(asctime)s %(filename)s:%(lineno)d] %(message)s",
+    )
+
+    support_matrix = SupportMatrix()
+    results = support_matrix.test_support_matrix()
+
+    # Always save results (now has a default output location)
+    support_matrix.save_results_to_csv(results, args.output)
+
+
+if __name__ == "__main__":
+    main()