[dynamo] Add NVIDIA Dynamo project for Forge

projects/caliper/prometheus_metrics/queries.yaml

@@ -301,3 +301,155 @@ queries:
     promql: DCGM_FI_DEV_GPU_TEMP
     unit: celsius
     description: GPU temperature
+
+  # ── Dynamo Frontend ─────────────────────────────────────────────────
+
+  dynamo_requests_total:
+    category: dynamo_frontend
+    promql: dynamo_frontend_requests_total{namespace=~"{ns}"}
+    unit: count
+    description: Total requests handled by Dynamo frontend
+
+  dynamo_request_rate:
+    category: dynamo_frontend
+    promql: >-
+      sum(rate(dynamo_frontend_requests_total
+      {namespace=~"{ns}"}[5m]))
+    unit: req/s
+    description: Dynamo frontend request rate
+
+  dynamo_input_token_rate:
+    category: dynamo_frontend
+    promql: >-
+      sum(rate(dynamo_frontend_input_sequence_tokens_sum
+      {namespace=~"{ns}"}[5m]))
+    unit: tokens/s
+    description: Dynamo frontend input token throughput
+
+  dynamo_output_token_rate:
+    category: dynamo_frontend
+    promql: >-
+      sum(rate(dynamo_frontend_output_sequence_tokens_sum
+      {namespace=~"{ns}"}[5m]))
+    unit: tokens/s
+    description: Dynamo frontend output token throughput
+
+  dynamo_queued_requests:
+    category: dynamo_frontend
+    promql: dynamo_frontend_queued_requests{namespace=~"{ns}"}
+    unit: count
+    description: Requests queued in Dynamo frontend
+
+  dynamo_ttft_avg:
+    category: dynamo_frontend
+    promql: >-
+      rate(dynamo_frontend_time_to_first_token_seconds_sum
+      {namespace=~"{ns}"}[5m])
+      / clamp_min(rate(dynamo_frontend_time_to_first_token_seconds_count
+      {namespace=~"{ns}"}[5m]), 1)
+    unit: seconds
+    description: Dynamo average time to first token
+
+  dynamo_itl_avg:
+    category: dynamo_frontend
+    promql: >-
+      rate(dynamo_frontend_inter_token_latency_seconds_sum
+      {namespace=~"{ns}"}[5m])
+      / clamp_min(rate(dynamo_frontend_inter_token_latency_seconds_count
+      {namespace=~"{ns}"}[5m]), 1)
+    unit: seconds
+    description: Dynamo average inter-token latency
+
+  dynamo_request_duration_avg:
+    category: dynamo_frontend
+    promql: >-
+      rate(dynamo_frontend_request_duration_seconds_sum
+      {namespace=~"{ns}"}[5m])
+      / clamp_min(rate(dynamo_frontend_request_duration_seconds_count
+      {namespace=~"{ns}"}[5m]), 1)
+    unit: seconds
+    description: Dynamo average request duration
+
+  dynamo_avg_input_tokens_per_request:
+    category: dynamo_frontend
+    promql: >-
+      rate(dynamo_frontend_input_sequence_tokens_sum
+      {namespace=~"{ns}"}[5m])
+      / clamp_min(rate(dynamo_frontend_input_sequence_tokens_count
+      {namespace=~"{ns}"}[5m]), 1)
+    unit: tokens
+    description: Average input tokens per request
+
+  dynamo_avg_output_tokens_per_request:
+    category: dynamo_frontend
+    promql: >-
+      rate(dynamo_frontend_output_sequence_tokens_sum
+      {namespace=~"{ns}"}[5m])
+      / clamp_min(rate(dynamo_frontend_output_sequence_tokens_count
+      {namespace=~"{ns}"}[5m]), 1)
+    unit: tokens
+    description: Average output tokens per request
+
+  # ── Dynamo Component ────────────────────────────────────────────────
+
+  dynamo_component_request_rate:
+    category: dynamo_component
+    promql: >-
+      sum by (dynamo_component) (rate(dynamo_component_requests_total
+      {namespace=~"{ns}"}[5m]))
+    unit: req/s
+    description: Request rate per Dynamo component
+
+  dynamo_component_request_duration_avg:
+    category: dynamo_component
+    promql: >-
+      sum by (dynamo_component) (
+        rate(dynamo_component_request_duration_seconds_sum
+        {namespace=~"{ns}"}[5m])
+        / clamp_min(rate(dynamo_component_request_duration_seconds_count
+        {namespace=~"{ns}"}[5m]), 1)
+      )
+    unit: seconds
+    description: Average request duration per Dynamo component
+
+  dynamo_component_request_bytes_rate:
+    category: dynamo_component
+    promql: >-
+      sum by (dynamo_component) (rate(dynamo_component_request_bytes_total
+      {namespace=~"{ns}"}[5m]))
+    unit: bytes/s
+    description: Request bytes throughput per Dynamo component
+
+  dynamo_component_response_bytes_rate:
+    category: dynamo_component
+    promql: >-
+      sum by (dynamo_component) (rate(dynamo_component_response_bytes_total
+      {namespace=~"{ns}"}[5m]))
+    unit: bytes/s
+    description: Response bytes throughput per Dynamo component
+
+  dynamo_gpu_cache_usage_percent:
+    category: dynamo_component
+    promql: dynamo_component_gpu_cache_usage_percent{namespace=~"{ns}"}
+    unit: percent
+    description: Dynamo GPU KV cache usage percentage
+
+  dynamo_total_blocks:
+    category: dynamo_component
+    promql: dynamo_component_total_blocks{namespace=~"{ns}"}
+    unit: count
+    description: Dynamo total KV cache blocks
+
+  # ── Dynamo Operator ─────────────────────────────────────────────────
+
+  dynamo_operator_reconcile_rate:
+    category: dynamo_operator
+    promql: sum(rate(dynamo_operator_reconcile_total[5m]))
+    unit: ops/s
+    description: Dynamo operator reconciliation rate
+
+  dynamo_operator_reconcile_errors:
+    category: dynamo_operator
+    promql: sum(rate(dynamo_operator_reconcile_errors_total[5m]))
+    unit: errors/s
+    description: Dynamo operator reconciliation error rate

projects/dynamo/orchestration/ci.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+Dynamo Project CI Operations
+"""
+
+import logging
+import types
+from pathlib import Path
+
+import click
+
+from projects.core.agentic.config_review import trigger_config_review_for_ci
+from projects.core.agentic.on_failure import agent_review_on_failure
+from projects.core.ci_entrypoint.fournos_resolve import create_fournos_resolve_entrypoint
+from projects.core.library import ci as ci_lib
+from projects.core.library import config, env, run, vault
+from projects.core.library.export import caliper_export_entrypoint
+from projects.core.library.replot import caliper_replot_entrypoint
+from projects.dynamo.orchestration.cleanup_phase import run as cleanup_toolbox_run
+from projects.dynamo.orchestration.preflight_phase import run as preflight_toolbox_run
+from projects.dynamo.orchestration.prepare_sequence import run_prepare_sequence
+from projects.dynamo.orchestration.test_phase import run as test_toolbox_run
+
+logger = logging.getLogger(__name__)
+
+
+def init():
+    """Initialize Dynamo orchestration environment"""
+    env.init()
+    run.init()
+    config.init(Path(__file__).parent)
+
+
+def list_vaults() -> list[str]:
+    """List all vaults (includes both mandatory and optional)."""
+    return vault.phase_vault_list_all()
+
+
+@click.group(cls=ci_lib.HelpfulGroup)
+@click.pass_context
+@ci_lib.safe_ci_function
+def main(ctx):
+    """Dynamo Project CI Operations for FORGE."""
+    ctx.ensure_object(types.SimpleNamespace)
+    init()
+
+    if ctx.invoked_subcommand == "resolve-fournos-config":
+        logger.info("No need to initialize the vaults for the resolve step")
+        return
+
+    vault.phase_vault_init(ctx.invoked_subcommand)
+
+
+@main.command()
+@click.pass_context
+@ci_lib.safe_ci_command
+@agent_review_on_failure
+def prepare(ctx) -> int:
+    """Prepare phase - Set up environment, operators, and Dynamo platform."""
+    return run_prepare_sequence()
+
+
+@main.command()
+@click.pass_context
+@ci_lib.safe_ci_command
+@agent_review_on_failure
+def preflight(ctx) -> int:
+    """Preflight check phase - Validate required Dynamo CRDs exist."""
+    return preflight_toolbox_run()
+
+
+@main.command()
+@click.pass_context
+@ci_lib.safe_ci_command
+@agent_review_on_failure
+def test(ctx) -> int:
+    """Test phase - Deploy DynamoGraphDeployment, run smoke test and benchmarks."""
+    trigger_config_review_for_ci(env.BASE_ARTIFACT_DIR, async_mode=True)
+    return test_toolbox_run()
+
+
+@main.command()
+@click.pass_context
+@ci_lib.safe_ci_command
+@agent_review_on_failure
+def pre_cleanup(ctx) -> int:
+    """Cleanup phase - Clean up Dynamo test resources."""
+    from projects.dynamo.orchestration import runtime_config
+
+    for run_spec in runtime_config.get_run_specs():
+        with runtime_config.activate_run_spec(run_spec):
+            cleanup_toolbox_run(namespace=run_spec.namespace)
+    return 0
+
+
+@main.command()
+@click.pass_context
+@ci_lib.safe_ci_command
+@agent_review_on_failure
+def post_cleanup(ctx) -> int:
+    """Cleanup phase - Final cleanup of Dynamo resources."""
+    from projects.dynamo.orchestration import runtime_config
+
+    for run_spec in runtime_config.get_run_specs():
+        with runtime_config.activate_run_spec(run_spec):
+            cleanup_toolbox_run(namespace=run_spec.namespace)
+    return 0
+
+
+main.add_command(create_fournos_resolve_entrypoint(vault_list_func=vault.phase_vault_list_all))
+main.add_command(caliper_export_entrypoint)
+main.add_command(caliper_replot_entrypoint)
+
+if __name__ == "__main__":
+    main()

projects/dynamo/orchestration/cleanup_phase.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import logging
+
+from projects.core.dsl.utils.k8s import oc_resource_exists
+from projects.dynamo.toolbox.cleanup_dynamo_resources.main import (
+    run as cleanup_dynamo_resources_toolbox_run,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def run(*, namespace: str | None = None) -> int:
+    """Delete Dynamo test resources from a namespace."""
+    from projects.dynamo.orchestration import runtime_config
+
+    if namespace is None:
+        namespace = runtime_config.get_namespace()
+
+    if not oc_resource_exists("namespace", namespace):
+        logger.info("Namespace %s does not exist, nothing to clean up", namespace)
+        return 0
+
+    benchmark_job_names = runtime_config.get_benchmark_job_names()
+    benchmark_name = benchmark_job_names[0] if benchmark_job_names else None
+
+    cleanup_dynamo_resources_toolbox_run(
+        namespace=namespace,
+        benchmark_job_name=benchmark_name,
+    )
+
+    return 0