ci: add harness integration CI for the four agent CLIs

.ci/harness.cloudbuild.yaml

@@ -0,0 +1,99 @@
+steps:
+  - id: pull-cache
+    name: 'gcr.io/cloud-builders/docker'
+    entrypoint: 'bash'
+    args: ['-c', 'docker pull us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache || exit 0']
+
+  - id: build-image
+    name: 'gcr.io/cloud-builders/docker'
+    waitFor: ['pull-cache']
+    args: ['build',
+           '--cache-from', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache',
+           '-t', 'evalbench-harness-ci',
+           '-t', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache',
+           '-f', 'evalbench_service/Dockerfile', '.']
+
+  # Pushed here rather than via top-level `images:`, which only runs when the
+  # whole build succeeds -- the failing builds would never warm the cache.
+  - id: push-cache
+    name: 'gcr.io/cloud-builders/docker'
+    waitFor: ['build-image']
+    allowFailure: true
+    args: ['push', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache']
+
+  - id: fetch-sa-key
+    name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+    entrypoint: 'bash'
+    waitFor: ['-']
+    args:
+      - '-c'
+      - |
+        mkdir -p /etc/evalbench-sa-key
+        gcloud secrets versions access latest --secret="evalbench-sa-key" --project="$_EVAL_PROJECT" > /etc/evalbench-sa-key/key.json
+    volumes:
+      - name: 'sa_key_volume'
+        path: '/etc/evalbench-sa-key'
+
+  - id: agy-cli
+    name: 'evalbench-harness-ci'
+    dir: '/evalbench'
+    waitFor: ['build-image', 'fetch-sa-key']
+    allowFailure: true
+    entrypoint: 'bash'
+    args: ['-c', 'CI_HARNESS=agy_cli uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml']
+    env: &harness_env
+      - 'EVAL_GCP_PROJECT_ID=${_EVAL_PROJECT}'
+      - 'EVAL_GCP_PROJECT_REGION=global'
+      - 'CI_CSQL_INSTANCE=${_CI_CSQL_INSTANCE}'
+      - 'UV_CACHE_DIR=/tmp/uv-cache'
+    volumes:
+      - name: 'eval_results'
+        path: '/evalbench/results'
+      - name: 'sa_key_volume'
+        path: '/etc/evalbench-sa-key'
+
+  - id: claude-code
+    name: 'evalbench-harness-ci'
+    dir: '/evalbench'
+    waitFor: ['build-image']
+    allowFailure: true
+    entrypoint: 'bash'
+    args: ['-c', 'CI_HARNESS=claude_code uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml']
+    env: *harness_env
+    volumes: &results_volume
+      - name: 'eval_results'
+        path: '/evalbench/results'
+
+  - id: codex-cli
+    name: 'evalbench-harness-ci'
+    dir: '/evalbench'
+    waitFor: ['build-image']
+    allowFailure: true
+    entrypoint: 'bash'
+    args: ['-c', 'CI_HARNESS=codex_cli uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml']
+    env: *harness_env
+    volumes: *results_volume
+
+  - id: gemini-cli
+    name: 'evalbench-harness-ci'
+    dir: '/evalbench'
+    waitFor: ['build-image']
+    allowFailure: true
+    entrypoint: 'bash'
+    args: ['-c', 'CI_HARNESS=gemini_cli uv run --no-sync evalbench/evalbench.py --experiment_config=.ci/run_config.yaml']
+    env: *harness_env
+    volumes: *results_volume
+
+  - id: verify-harnesses
+    name: 'evalbench-harness-ci'
+    dir: '/evalbench'
+    waitFor: ['agy-cli', 'claude-code', 'codex-cli', 'gemini-cli']
+    args: ['uv', 'run', '--no-sync', 'python3', '.ci/verify_harness.py']
+    volumes: *results_volume
+
+timeout: '3600s'
+
+options:
+  logging: CLOUD_LOGGING_ONLY
+  # Sized for the four concurrent harness steps, not for the build.
+  machineType: 'E2_HIGHCPU_8'

.ci/harness_smoke.evalset.json

@@ -0,0 +1,31 @@
+{
+  "scenarios": [
+    {
+      "id": "ci-csql-list-and-get",
+      "starting_prompt": "list all instances in project ${EVAL_GCP_PROJECT_ID}",
+      "conversation_plan": "The user wants to see every Cloud SQL instance in project ${EVAL_GCP_PROJECT_ID}, then check the health of one of them. Once the agent has listed the instances, the user asks for the current state of ${CI_CSQL_INSTANCE} and expects the agent to report whether it is RUNNABLE. The user accepts that answer and ends the conversation.",
+      "expected_trajectory": [
+        "cloud-sql__list_instances",
+        "cloud-sql__get_instance"
+      ],
+      "env": {
+        "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}"
+      },
+      "kind": "tools",
+      "max_turns": 3
+    },
+    {
+      "id": "ci-csql-get-instance-not-found",
+      "starting_prompt": "Get the details for the Cloud SQL instance named 'ci-missing-instance-do-not-create' in project '${EVAL_GCP_PROJECT_ID}'.",
+      "conversation_plan": "The user wants details for an instance that does not exist. The agent should call get_instance once, receive a not-found error, and report to the user that the instance could not be found. The user accepts that answer and ends the conversation. The user must not ask the agent to list instances, search for similar names, or otherwise recover.",
+      "expected_trajectory": [
+        "cloud-sql__get_instance"
+      ],
+      "env": {
+        "GOOGLE_CLOUD_PROJECT": "${EVAL_GCP_PROJECT_ID}"
+      },
+      "kind": "tools",
+      "max_turns": 2
+    }
+  ]
+}

.ci/model_configs/agy_cli_model.yaml

@@ -0,0 +1,20 @@
+generator: agy_cli
+# ADC does not authorize the Pro (High) tier.
+model: "Gemini 3.1 Pro (Low)"
+timeout: "20m"
+
+env:
+  GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID}
+  GOOGLE_CLOUD_LOCATION: "global"
+  GOOGLE_GENAI_USE_VERTEXAI: "true"
+
+setup:
+  mcp_servers:
+    "cloud-sql":
+      httpUrl: "https://sqladmin.googleapis.com/mcp"
+      authProviderType: google_credentials
+      oauth:
+        scopes:
+        - https://www.googleapis.com/auth/cloud-platform
+      headers:
+        X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID}

.ci/model_configs/claude_code_model.yaml

@@ -0,0 +1,23 @@
+claude_code_version: "@anthropic-ai/claude-code@2.1.85"
+generator: claude_code
+model: "claude-opus-4-6"
+
+use_vertex: true
+vertex_project_id: !ENV ${EVAL_GCP_PROJECT_ID}
+# Not us-central1: Vertex serves Claude from only a few regions.
+vertex_region: "us-east5"
+
+env:
+  GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID}
+  GOOGLE_CLOUD_LOCATION: "us-central1"
+
+setup:
+  mcp_servers:
+    "cloud-sql":
+      httpUrl: "https://sqladmin.googleapis.com/mcp"
+      authProviderType: google_credentials
+      oauth:
+        scopes:
+        - https://www.googleapis.com/auth/cloud-platform
+      headers:
+        X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID}

.ci/model_configs/codex_cli_model.yaml

@@ -0,0 +1,23 @@
+codex_cli_version: "@openai/codex@latest"
+generator: codex_cli
+model: "gpt-5.5"
+
+openai_api_key_secret: !ENV projects/${EVAL_GCP_PROJECT_ID}/secrets/openai-api-key/versions/2
+
+# Codex reports token counts but no cost; these rates turn them into cost_usd.
+pricing:
+  input_per_million_usd:        1.25
+  cached_input_per_million_usd: 0.125
+  output_per_million_usd:       10.0
+
+env:
+  GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID}
+  GOOGLE_CLOUD_LOCATION: "us-central1"
+
+setup:
+  mcp_servers:
+    "cloud-sql":
+      httpUrl: "https://sqladmin.googleapis.com/mcp"
+      authProviderType: google_credentials
+      headers:
+        X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID}

.ci/model_configs/gemini_cli_model.yaml

@@ -0,0 +1,19 @@
+gemini_cli_version: "@google/gemini-cli@0.55.1"
+generator: gemini_cli
+
+env:
+  GOOGLE_CLOUD_PROJECT: !ENV ${EVAL_GCP_PROJECT_ID}
+  GOOGLE_CLOUD_LOCATION: "global"
+  GOOGLE_GENAI_USE_VERTEXAI: "true"
+  GEMINI_CLI_TRUST_WORKSPACE: "true"
+
+setup:
+  mcp_servers:
+    "cloud-sql":
+      httpUrl: "https://sqladmin.googleapis.com/mcp"
+      authProviderType: google_credentials
+      oauth:
+        scopes:
+        - https://www.googleapis.com/auth/cloud-platform
+      headers:
+        X-Goog-User-Project: !ENV ${EVAL_GCP_PROJECT_ID}

.ci/run_config.yaml

@@ -0,0 +1,28 @@
+# Shared by all four harnesses. CI_HARNESS selects which one runs, so every
+# harness is scored identically and stays comparable.
+dataset_config: .ci/harness_smoke.evalset.json
+dataset_format: agent-format
+
+orchestrator: agent
+model_config: !ENV .ci/model_configs/${CI_HARNESS}_model.yaml
+simulated_user_model_config: datasets/model_configs/gemini_3.1_pro_model.yaml
+
+scorers:
+  trajectory_matcher: {}
+  turn_count: {}
+  agent_steps: {}
+  end_to_end_latency: {}
+  tool_call_latency: {}
+  token_consumption: {}
+  tokens_processed: {}
+  effective_billed_tokens: {}
+  goal_completion:
+    model_config: datasets/model_configs/gemini_3.1_pro_model.yaml
+  behavioral_metrics:
+    model_config: datasets/model_configs/gemini_3.1_pro_model.yaml
+  parameter_analysis:
+    model_config: datasets/model_configs/gemini_3.1_pro_model.yaml
+
+reporting:
+  csv:
+    output_directory: !ENV results/ci/${CI_HARNESS}