From 8b2d5029663f55bf7ca40b13d5ba7c66d644ffda Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 18 Aug 2026 10:44:38 -0400 Subject: [PATCH 1/4] Add cpt pipeline for rhaiis via fournos ui --- .../rhaiis/orchestration/presets.d/cpt.yaml | 117 ++++++++++++++++++ .../orchestration/presets.d/presets.yaml | 1 + 2 files changed, 118 insertions(+) create mode 100644 projects/rhaiis/orchestration/presets.d/cpt.yaml diff --git a/projects/rhaiis/orchestration/presets.d/cpt.yaml b/projects/rhaiis/orchestration/presets.d/cpt.yaml new file mode 100644 index 000000000..16353a0ee --- /dev/null +++ b/projects/rhaiis/orchestration/presets.d/cpt.yaml @@ -0,0 +1,117 @@ +# CPT Pipeline Definitions +# Forge repo path: projects/rhaiis/orchestration/presets.d/cpt.yaml +# +# Each top-level key (except __cpt) defines one CPT pipeline. +# __models: keys are preset names, values are per-model config overrides (null = global only). +# Non-__ keys at the pipeline level are global overrides applied to every job. + +__cpt: true + +# ────────────────────────────────────────────── +# vLLM Release — Release regression testing models +# ────────────────────────────────────────────── +cpt-vllm-release: + __description: "vLLM Release CPT — Tier 1 models across all workload profiles" + __engine: vllm + __models: + llama-70b: + llama-70b-bf16: + granite-8b: + # Per-model overrides — these merge on top of the global overrides below + rhaiis.engines.vllm.args.gpu-memory-utilization: 0.95 + granite-8b-bf16: + mistral-24b: + mistral-24b-bf16: + qwen25-7b: + qwen25-7b-bf16: + __workloads: + - ci-quick + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +# ────────────────────────────────────────────── +# SGLang Release — Tier 1 models +# ────────────────────────────────────────────── +cpt-sglang-competitive: + __description: "SGLang Release CPT — Tier 1 models across all workload profiles" + __engine: sglang + __models: + llama-70b: + llama-70b-bf16: + granite-8b: + granite-8b-bf16: + mistral-24b: + mistral-24b-bf16: + qwen25-7b: + qwen25-7b-bf16: + __workloads: + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +# ────────────────────────────────────────────── +# TensorRT-LLM Release — NVIDIA only +# ────────────────────────────────────────────── +cpt-trtllm-competitive: + __description: "TensorRT-LLM Release CPT — supported NVIDIA models across all workload profiles" + __engine: trtllm + __accelerator: nvidia + __models: + llama-70b: + granite-8b: + mistral-24b: + qwen25-7b: + __workloads: + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +# ────────────────────────────────────────────── +# vLLM Competitive — wider model coverage +# ────────────────────────────────────────────── +cpt-vllm-competitive: + __description: "vLLM Competitive CPT — wide model coverage for cross-engine comparison" + __engine: vllm + __models: + llama-70b: + llama-70b-bf16: + granite-8b: + granite-8b-bf16: + mistral-24b: + mistral-24b-bf16: + qwen25-7b: + qwen25-7b-bf16: + llama-8b: + llama-4-scout: + llama-4-maverick: + qwen3-235b: + deepseek-r1: + __workloads: + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false diff --git a/projects/rhaiis/orchestration/presets.d/presets.yaml b/projects/rhaiis/orchestration/presets.d/presets.yaml index c48f3203a..61d50bf67 100644 --- a/projects/rhaiis/orchestration/presets.d/presets.yaml +++ b/projects/rhaiis/orchestration/presets.d/presets.yaml @@ -30,6 +30,7 @@ ci-test: caliper.postprocess.csv_dashboard.enabled: false tests.rhaiis.slack_notify_always: true rhaiis.agent_analysis.enabled: false + tests.rhaiis.run_benchmark: true ci-quick: tests.rhaiis.workload_key: ci-quick From c0b6f902f72a074f169f61189438c3a6d5e74d9d Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 18 Aug 2026 12:23:20 -0400 Subject: [PATCH 2/4] move cpt.yaml into cpt.d from presets.d folder --- projects/rhaiis/orchestration/{presets.d => cpt.d}/cpt.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename projects/rhaiis/orchestration/{presets.d => cpt.d}/cpt.yaml (100%) diff --git a/projects/rhaiis/orchestration/presets.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml similarity index 100% rename from projects/rhaiis/orchestration/presets.d/cpt.yaml rename to projects/rhaiis/orchestration/cpt.d/cpt.yaml From a58f22f3bbcda0bd3d753025f41a002519b0ca02 Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Tue, 18 Aug 2026 23:47:14 -0400 Subject: [PATCH 3/4] upddate slack notification mechanism --- projects/rhaiis/orchestration/cpt.d/cpt.yaml | 50 ++++++++++++++++---- projects/rhaiis/postprocess/regression.py | 35 +++++++------- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml index 16353a0ee..63dc91387 100644 --- a/projects/rhaiis/orchestration/cpt.d/cpt.yaml +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -1,29 +1,40 @@ # CPT Pipeline Definitions -# Forge repo path: projects/rhaiis/orchestration/presets.d/cpt.yaml +# Forge repo path: projects/rhaiis/orchestration/cpt.d/cpt.yaml # # Each top-level key (except __cpt) defines one CPT pipeline. # __models: keys are preset names, values are per-model config overrides (null = global only). +# __tp: GPU count (tensor-parallel-size) for Kueue admission. Falls back to Forge's models.yaml. # Non-__ keys at the pipeline level are global overrides applied to every job. __cpt: true # ────────────────────────────────────────────── -# vLLM Release — Release regression testing models +# vLLM Release — Tier 1 models # ────────────────────────────────────────────── cpt-vllm-release: __description: "vLLM Release CPT — Tier 1 models across all workload profiles" __engine: vllm __models: - llama-70b: - llama-70b-bf16: - granite-8b: - # Per-model overrides — these merge on top of the global overrides below - rhaiis.engines.vllm.args.gpu-memory-utilization: 0.95 + llama-70b/tp4: + __tp: 4 + llama-70b/tp2: + __tp: 2 + rhaiis.engines.vllm.args.tensor-parallel-size: 2 + llama-70b-bf16/tp4: + __tp: 4 + llama-70b-bf16/tp2: + __tp: 2 + rhaiis.engines.vllm.args.tensor-parallel-size: 2 granite-8b-bf16: + __tp: 1 mistral-24b: + __tp: 1 mistral-24b-bf16: + __tp: 1 qwen25-7b: + __tp: 1 qwen25-7b-bf16: + __tp: 1 __workloads: - ci-quick - profile1 @@ -44,13 +55,19 @@ cpt-sglang-competitive: __engine: sglang __models: llama-70b: + __tp: 4 llama-70b-bf16: - granite-8b: + __tp: 4 granite-8b-bf16: + __tp: 1 mistral-24b: + __tp: 1 mistral-24b-bf16: + __tp: 1 qwen25-7b: + __tp: 1 qwen25-7b-bf16: + __tp: 1 __workloads: - profile1 - profile2 @@ -71,9 +88,11 @@ cpt-trtllm-competitive: __accelerator: nvidia __models: llama-70b: - granite-8b: + __tp: 4 mistral-24b: + __tp: 1 qwen25-7b: + __tp: 1 __workloads: - profile1 - profile2 @@ -93,18 +112,29 @@ cpt-vllm-competitive: __engine: vllm __models: llama-70b: + __tp: 4 llama-70b-bf16: - granite-8b: + __tp: 4 granite-8b-bf16: + __tp: 1 mistral-24b: + __tp: 1 mistral-24b-bf16: + __tp: 1 qwen25-7b: + __tp: 1 qwen25-7b-bf16: + __tp: 1 llama-8b: + __tp: 1 llama-4-scout: + __tp: 2 llama-4-maverick: + __tp: 8 qwen3-235b: + __tp: 4 deepseek-r1: + __tp: 8 __workloads: - profile1 - profile2 diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index dfb735626..76471c822 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -23,6 +23,20 @@ RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" +_SLACK_USER_RE = re.compile(r"^[UW][A-Z0-9]+$") +_SLACK_GROUP_RE = re.compile(r"^S[A-Z0-9]+$") + + +def _format_slack_user_line(slack_user: str) -> str: + """Build the 'Triggered by' line, supporting both user and group IDs.""" + if not slack_user: + return "" + if _SLACK_USER_RE.match(slack_user): + return f"*Triggered by:* <@{slack_user}>\n" + if _SLACK_GROUP_RE.match(slack_user): + return f"*Triggered by:* \n" + return f"*Triggered by:* {slack_user}\n" + def _send_via_topsail_bot( message: str, *, notification_vault: str | None = None, channel_id: str | None = None @@ -523,12 +537,7 @@ def send_regression_notification( details = "\n".join(detail_lines) - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" + user_line = _format_slack_user_line(slack_user) report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" @@ -597,12 +606,7 @@ def send_success_notification( Returns: True if notification sent successfully """ - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" + user_line = _format_slack_user_line(slack_user) parallelism_parts = [] if tp: @@ -699,12 +703,7 @@ def send_failure_notification( Returns: True if notification sent successfully """ - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" + user_line = _format_slack_user_line(slack_user) parallelism_parts = [] if tp: From 74ff52431fecef218baf8ed9a9c2866bdbc09cff Mon Sep 17 00:00:00 2001 From: Harshith-umesh Date: Wed, 19 Aug 2026 14:58:36 -0400 Subject: [PATCH 4/4] upddate cpt.yaml format --- projects/rhaiis/orchestration/cpt.d/cpt.yaml | 90 +++++++------------- 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml index 63dc91387..3ab276e03 100644 --- a/projects/rhaiis/orchestration/cpt.d/cpt.yaml +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -2,8 +2,9 @@ # Forge repo path: projects/rhaiis/orchestration/cpt.d/cpt.yaml # # Each top-level key (except __cpt) defines one CPT pipeline. -# __models: keys are preset names, values are per-model config overrides (null = global only). -# __tp: GPU count (tensor-parallel-size) for Kueue admission. Falls back to Forge's models.yaml. +# __models: keys use the format /tp where N is the tensor-parallel-size (GPU count). +# Values are per-model config overrides (null = global only). +# The /tpN suffix is parsed automatically — no separate __tp field needed. # Non-__ keys at the pipeline level are global overrides applied to every job. __cpt: true @@ -16,25 +17,16 @@ cpt-vllm-release: __engine: vllm __models: llama-70b/tp4: - __tp: 4 llama-70b/tp2: - __tp: 2 rhaiis.engines.vllm.args.tensor-parallel-size: 2 llama-70b-bf16/tp4: - __tp: 4 llama-70b-bf16/tp2: - __tp: 2 rhaiis.engines.vllm.args.tensor-parallel-size: 2 - granite-8b-bf16: - __tp: 1 - mistral-24b: - __tp: 1 - mistral-24b-bf16: - __tp: 1 - qwen25-7b: - __tp: 1 - qwen25-7b-bf16: - __tp: 1 + granite-8b-bf16/tp1: + mistral-24b/tp1: + mistral-24b-bf16/tp1: + qwen25-7b/tp1: + qwen25-7b-bf16/tp1: __workloads: - ci-quick - profile1 @@ -54,20 +46,13 @@ cpt-sglang-competitive: __description: "SGLang Release CPT — Tier 1 models across all workload profiles" __engine: sglang __models: - llama-70b: - __tp: 4 - llama-70b-bf16: - __tp: 4 - granite-8b-bf16: - __tp: 1 - mistral-24b: - __tp: 1 - mistral-24b-bf16: - __tp: 1 - qwen25-7b: - __tp: 1 - qwen25-7b-bf16: - __tp: 1 + llama-70b/tp4: + llama-70b-bf16/tp4: + granite-8b-bf16/tp1: + mistral-24b/tp1: + mistral-24b-bf16/tp1: + qwen25-7b/tp1: + qwen25-7b-bf16/tp1: __workloads: - profile1 - profile2 @@ -87,12 +72,9 @@ cpt-trtllm-competitive: __engine: trtllm __accelerator: nvidia __models: - llama-70b: - __tp: 4 - mistral-24b: - __tp: 1 - qwen25-7b: - __tp: 1 + llama-70b/tp4: + mistral-24b/tp1: + qwen25-7b/tp1: __workloads: - profile1 - profile2 @@ -111,30 +93,18 @@ cpt-vllm-competitive: __description: "vLLM Competitive CPT — wide model coverage for cross-engine comparison" __engine: vllm __models: - llama-70b: - __tp: 4 - llama-70b-bf16: - __tp: 4 - granite-8b-bf16: - __tp: 1 - mistral-24b: - __tp: 1 - mistral-24b-bf16: - __tp: 1 - qwen25-7b: - __tp: 1 - qwen25-7b-bf16: - __tp: 1 - llama-8b: - __tp: 1 - llama-4-scout: - __tp: 2 - llama-4-maverick: - __tp: 8 - qwen3-235b: - __tp: 4 - deepseek-r1: - __tp: 8 + llama-70b/tp4: + llama-70b-bf16/tp4: + granite-8b-bf16/tp1: + mistral-24b/tp1: + mistral-24b-bf16/tp1: + qwen25-7b/tp1: + qwen25-7b-bf16/tp1: + llama-8b/tp1: + llama-4-scout/tp2: + llama-4-maverick/tp8: + qwen3-235b/tp4: + deepseek-r1/tp8: __workloads: - profile1 - profile2