diff --git a/projects/rhaiis/orchestration/ci.py b/projects/rhaiis/orchestration/ci.py index 0e76806f0..3508cdefd 100755 --- a/projects/rhaiis/orchestration/ci.py +++ b/projects/rhaiis/orchestration/ci.py @@ -9,9 +9,11 @@ import prepare_rhaiis import test_rhaiis +from projects.core.agentic.config_review import trigger_config_review_for_ci +from projects.core.agentic.on_failure import agent_review_on_failure from projects.core.ci_entrypoint.fournos_resolve import create_fournos_resolve_entrypoint from projects.core.library import ci as ci_lib -from projects.core.library import vault +from projects.core.library import env, vault from projects.core.library.export import caliper_export_entrypoint from projects.rhaiis.orchestration import runtime_config @@ -139,8 +141,10 @@ def prepare(ctx): @main.command() @click.pass_context @ci_lib.safe_ci_entrypoint +@agent_review_on_failure def test(ctx): """Test phase - Deploy model, run benchmarks, capture results.""" + trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True) return test_rhaiis.test() diff --git a/projects/rhaiis/orchestration/config.d/models.yaml b/projects/rhaiis/orchestration/config.d/models.yaml index 59739985c..ef50d186b 100644 --- a/projects/rhaiis/orchestration/config.d/models.yaml +++ b/projects/rhaiis/orchestration/config.d/models.yaml @@ -5,6 +5,13 @@ qwen3-0_6b: vllm_args: tensor-parallel-size: 1 +# Dashboard override stub (set hf_model_id and vllm_args at runtime) +custom: + name: custom + hf_model_id: "" + vllm_args: + tensor-parallel-size: 1 + # Llama-4 Scout (MoE) llama-4-scout: name: Llama-4-Scout-17B-16E-Instruct diff --git a/projects/rhaiis/orchestration/config.d/workloads.yaml b/projects/rhaiis/orchestration/config.d/workloads.yaml index c03e32bc2..b1c277004 100644 --- a/projects/rhaiis/orchestration/config.d/workloads.yaml +++ b/projects/rhaiis/orchestration/config.d/workloads.yaml @@ -7,25 +7,21 @@ profile1: data: "prompt_tokens=1000,output_tokens=1000" rates: [1,50,100,200,300] max_seconds: 450 - rampup: 10 profile2: data: "prompt_tokens=512,prompt_tokens_stdev=128,prompt_tokens_min=1,prompt_tokens_max=1024,output_tokens=2048,output_tokens_stdev=512,output_tokens_min=1,output_tokens_max=4096" rates: [1,50,100,200,300] max_seconds: 450 - rampup: 10 profile3: data: "prompt_tokens=2048,output_tokens=128" rates: [1, 50, 100, 200, 300] max_seconds: 450 - rampup: 10 profile4: data: "prompt_tokens=8000,output_tokens=1000" rates: [1,25,50,75,100] max_seconds: 450 - rampup: 10 samples: 50 profile5: @@ -44,3 +40,8 @@ profile7: rates: [1, 50, 100, 200, 300] max_seconds: 450 +# Dashboard override stub (set data, rates, max_seconds at runtime) +custom: + data: "prompt_tokens=1000,output_tokens=1000" + rates: [1] + max_seconds: 450 diff --git a/projects/rhaiis/orchestration/config.yaml b/projects/rhaiis/orchestration/config.yaml index dd7206936..320a42cf3 100644 --- a/projects/rhaiis/orchestration/config.yaml +++ b/projects/rhaiis/orchestration/config.yaml @@ -3,6 +3,7 @@ vaults: - psap-forge-mlflow-export - psap-forge-aws-s3-export - psap-forge-dashboard-s3 +- psap-models-corp-rh benchmarks: guidellm: @@ -77,3 +78,11 @@ caliper: tags: {} parameters: {} metrics: {} + +agentic: + enabled: true + model_key: qwen-3-6-35b + on_failure: + enabled: true + config_review: + enabled: true