diff --git a/projects/rhaiis/orchestration/cpt.d/cpt.yaml b/projects/rhaiis/orchestration/cpt.d/cpt.yaml new file mode 100644 index 000000000..3ab276e03 --- /dev/null +++ b/projects/rhaiis/orchestration/cpt.d/cpt.yaml @@ -0,0 +1,117 @@ +# CPT Pipeline Definitions +# Forge repo path: projects/rhaiis/orchestration/cpt.d/cpt.yaml +# +# Each top-level key (except __cpt) defines one CPT pipeline. +# __models: keys use the format /tp where N is the tensor-parallel-size (GPU count). +# Values are per-model config overrides (null = global only). +# The /tpN suffix is parsed automatically — no separate __tp field needed. +# Non-__ keys at the pipeline level are global overrides applied to every job. + +__cpt: true + +# ────────────────────────────────────────────── +# vLLM Release — Tier 1 models +# ────────────────────────────────────────────── +cpt-vllm-release: + __description: "vLLM Release CPT — Tier 1 models across all workload profiles" + __engine: vllm + __models: + llama-70b/tp4: + llama-70b/tp2: + rhaiis.engines.vllm.args.tensor-parallel-size: 2 + llama-70b-bf16/tp4: + llama-70b-bf16/tp2: + rhaiis.engines.vllm.args.tensor-parallel-size: 2 + granite-8b-bf16/tp1: + mistral-24b/tp1: + mistral-24b-bf16/tp1: + qwen25-7b/tp1: + qwen25-7b-bf16/tp1: + __workloads: + - ci-quick + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +# ────────────────────────────────────────────── +# SGLang Release — Tier 1 models +# ────────────────────────────────────────────── +cpt-sglang-competitive: + __description: "SGLang Release CPT — Tier 1 models across all workload profiles" + __engine: sglang + __models: + llama-70b/tp4: + llama-70b-bf16/tp4: + granite-8b-bf16/tp1: + mistral-24b/tp1: + mistral-24b-bf16/tp1: + qwen25-7b/tp1: + qwen25-7b-bf16/tp1: + __workloads: + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +# ────────────────────────────────────────────── +# TensorRT-LLM Release — NVIDIA only +# ────────────────────────────────────────────── +cpt-trtllm-competitive: + __description: "TensorRT-LLM Release CPT — supported NVIDIA models across all workload profiles" + __engine: trtllm + __accelerator: nvidia + __models: + llama-70b/tp4: + mistral-24b/tp1: + qwen25-7b/tp1: + __workloads: + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false + +# ────────────────────────────────────────────── +# vLLM Competitive — wider model coverage +# ────────────────────────────────────────────── +cpt-vllm-competitive: + __description: "vLLM Competitive CPT — wide model coverage for cross-engine comparison" + __engine: vllm + __models: + llama-70b/tp4: + llama-70b-bf16/tp4: + granite-8b-bf16/tp1: + mistral-24b/tp1: + mistral-24b-bf16/tp1: + qwen25-7b/tp1: + qwen25-7b-bf16/tp1: + llama-8b/tp1: + llama-4-scout/tp2: + llama-4-maverick/tp8: + qwen3-235b/tp4: + deepseek-r1/tp8: + __workloads: + - profile1 + - profile2 + - profile3 + - profile4 + rhaiis.profiler.enabled: true + tests.rhaiis.run_benchmark: true + caliper.postprocess.csv_dashboard.enabled: true + tests.rhaiis.slack_notify_always: true + rhaiis.agent_analysis.enabled: false diff --git a/projects/rhaiis/orchestration/presets.d/presets.yaml b/projects/rhaiis/orchestration/presets.d/presets.yaml index 201ca4db6..c7d573295 100644 --- a/projects/rhaiis/orchestration/presets.d/presets.yaml +++ b/projects/rhaiis/orchestration/presets.d/presets.yaml @@ -30,6 +30,7 @@ ci-test: caliper.postprocess.csv_dashboard.enabled: false tests.rhaiis.slack_notify_always: true rhaiis.agent_analysis.enabled: false + tests.rhaiis.run_benchmark: true ci-quick: tests.rhaiis.workload_key: ci-quick diff --git a/projects/rhaiis/postprocess/regression.py b/projects/rhaiis/postprocess/regression.py index dfb735626..76471c822 100644 --- a/projects/rhaiis/postprocess/regression.py +++ b/projects/rhaiis/postprocess/regression.py @@ -23,6 +23,20 @@ RHAIIS_SLACK_CHANNEL_ID = "C0B9T6JUW74" +_SLACK_USER_RE = re.compile(r"^[UW][A-Z0-9]+$") +_SLACK_GROUP_RE = re.compile(r"^S[A-Z0-9]+$") + + +def _format_slack_user_line(slack_user: str) -> str: + """Build the 'Triggered by' line, supporting both user and group IDs.""" + if not slack_user: + return "" + if _SLACK_USER_RE.match(slack_user): + return f"*Triggered by:* <@{slack_user}>\n" + if _SLACK_GROUP_RE.match(slack_user): + return f"*Triggered by:* \n" + return f"*Triggered by:* {slack_user}\n" + def _send_via_topsail_bot( message: str, *, notification_vault: str | None = None, channel_id: str | None = None @@ -523,12 +537,7 @@ def send_regression_notification( details = "\n".join(detail_lines) - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" + user_line = _format_slack_user_line(slack_user) report_line = f"*Agent Analysis:* <{report_url}|View Report>\n" if report_url else "" @@ -597,12 +606,7 @@ def send_success_notification( Returns: True if notification sent successfully """ - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" + user_line = _format_slack_user_line(slack_user) parallelism_parts = [] if tp: @@ -699,12 +703,7 @@ def send_failure_notification( Returns: True if notification sent successfully """ - if slack_user and re.match(r"^[UW][A-Z0-9]+$", slack_user): - user_line = f"*Triggered by:* <@{slack_user}>\n" - elif slack_user: - user_line = f"*Triggered by:* {slack_user}\n" - else: - user_line = "" + user_line = _format_slack_user_line(slack_user) parallelism_parts = [] if tp: