Skip to content

Commit 2c6ef86

Browse files
refactor: remove inline comments from CI configuration and update cloud SQL smoke test conversation plan
1 parent b41f45e commit 2c6ef86

2 files changed

Lines changed: 1 addition & 9 deletions

File tree

‎.ci/harness.cloudbuild.yaml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
11
steps:
2-
# 1. Pull cached Docker image
32
- id: pull-cache
43
name: 'gcr.io/cloud-builders/docker'
54
entrypoint: 'bash'
65
args: ['-c', 'docker pull us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache || exit 0']
76

8-
# 2. Build Docker image
97
- id: build-image
108
name: 'gcr.io/cloud-builders/docker'
119
waitFor: ['pull-cache']
@@ -23,7 +21,6 @@ steps:
2321
allowFailure: true
2422
args: ['push', 'us-central1-docker.pkg.dev/$PROJECT_ID/evalbench/evalbench-harness-ci:cache']
2523

26-
# 3. Fetch SA key ONLY for agy-cli
2724
- id: fetch-sa-key
2825
name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
2926
entrypoint: 'bash'
@@ -37,7 +34,6 @@ steps:
3734
- name: 'sa_key_volume'
3835
path: '/etc/evalbench-sa-key'
3936

40-
# 4. agy-cli: mounts results_volume + sa_key_volume
4137
- id: agy-cli
4238
name: 'evalbench-harness-ci'
4339
dir: '/evalbench'
@@ -56,7 +52,6 @@ steps:
5652
- name: 'sa_key_volume'
5753
path: '/etc/evalbench-sa-key'
5854

59-
# 5. claude-code: only requires results_volume
6055
- id: claude-code
6156
name: 'evalbench-harness-ci'
6257
dir: '/evalbench'
@@ -69,7 +64,6 @@ steps:
6964
- name: 'eval_results'
7065
path: '/evalbench/results'
7166

72-
# 6. codex-cli: only requires results_volume
7367
- id: codex-cli
7468
name: 'evalbench-harness-ci'
7569
dir: '/evalbench'
@@ -80,7 +74,6 @@ steps:
8074
env: *harness_env
8175
volumes: *results_volume
8276

83-
# 7. gemini-cli: only requires results_volume
8477
- id: gemini-cli
8578
name: 'evalbench-harness-ci'
8679
dir: '/evalbench'
@@ -91,7 +84,6 @@ steps:
9184
env: *harness_env
9285
volumes: *results_volume
9386

94-
# 8. verify-harnesses
9587
- id: verify-harnesses
9688
name: 'evalbench-harness-ci'
9789
dir: '/evalbench'

‎.ci/harness_smoke.evalset.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
{
44
"id": "ci-csql-list-and-get",
55
"starting_prompt": "list all instances in project ${EVAL_GCP_PROJECT_ID}",
6-
"conversation_plan": "Ask the agent to list instances in project ${EVAL_GCP_PROJECT_ID}. Once all instances are listed if ${CI_CSQL_INSTANCE} exist get its state and validate its RUNNABLE",
6+
"conversation_plan": "The user wants to see every Cloud SQL instance in project ${EVAL_GCP_PROJECT_ID}, then check the health of one of them. Once the agent has listed the instances, the user asks for the current state of ${CI_CSQL_INSTANCE} and expects the agent to report whether it is RUNNABLE. The user accepts that answer and ends the conversation.",
77
"expected_trajectory": [
88
"cloud-sql__list_instances",
99
"cloud-sql__get_instance"

0 commit comments

Comments
 (0)