From ef5a2d0f308ea1672a79f6cc0b54b04c52bc7b12 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 05:38:13 +0000 Subject: [PATCH 01/13] ci: update evalbench pipeline trigger and add missing scorers - **Pipeline Alignment:** Updated `cloudbuild.yaml` trigger logic to support the `ci:run-evals` label for non-release branches and set `RELEASE_VERSION` context accordingly. - **Scorers:** Added missing `skills_best_practices` and `skills_trajectory` evaluation scorers to `evals/run_config.yaml`. --- .github/labels.yaml | 4 ++++ cloudbuild.yaml | 26 +++++++++++++++----------- evals/run_config.yaml | 4 ++++ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/.github/labels.yaml b/.github/labels.yaml index 4ed8706..636f768 100644 --- a/.github/labels.yaml +++ b/.github/labels.yaml @@ -84,3 +84,7 @@ - name: 'release-please:force-run' color: bdca82 description: Manually trigger the release please workflow on a PR. + +- name: 'ci:run-evals' + color: 4285f4 + description: Manually trigger the evaluation CI pipeline on a PR. diff --git a/cloudbuild.yaml b/cloudbuild.yaml index ed2378a..6366891 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,12 +27,7 @@ steps: - | set -e - # Only run on release branches - if [[ "$_HEAD_BRANCH" != release-please-* ]]; then - echo "Not a release-please branch. Exiting." - exit 0 - fi - echo "Release branch detected. Fetching PR data from GitHub API..." + echo "Fetching PR data from GitHub API..." # Fetch PR data and status code HTTP_STATUS=$(curl -s -o pr_data.json -w "%{http_code}" -H "Authorization: token $$GITHUB_TOKEN" \ @@ -50,25 +45,34 @@ steps: PR_LABELS=$(echo "$$PR_DATA" | jq -r '[.labels[].name] | join(",")') PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') - # Determine Release Version (Use double quotes and $$ for bash variables) - if [[ "$$PR_LABELS" == *"autorelease: pending"* ]]; then + # Check if execution labels are present + if [[ "$$PR_LABELS" != *"autorelease: pending"* && "$$PR_LABELS" != *"ci:run-evals"* ]]; then + echo "PR does not have 'autorelease: pending' or 'ci:run-evals' label. Skipping execution." + exit 0 + fi + echo "Execution label detected. Processing release version context..." + + # Determine Release Version based on branch name + if [[ "$_HEAD_BRANCH" == release-please-* ]]; then if [[ "$$PR_TITLE" =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then export RELEASE_VERSION="$${BASH_REMATCH[1]}" else - export RELEASE_VERSION="unknown" + export RELEASE_VERSION="pr-$_PR_NUMBER-release-unknown" fi else - export RELEASE_VERSION="unknown" + export RELEASE_VERSION="pr-$_PR_NUMBER-ci-run-evals" fi # Workaround for evalbench bug: settings are only applied if path basename matches extension ID ln -s /workspace /workspace/alloydb cd /evalbench + # Evalbench environment variables export EVAL_GCP_PROJECT_ID=$PROJECT_ID export EVAL_GCP_PROJECT_REGION=$_ALLOYDB_REGION export EVAL_REPORTING_PROJECT=$_EVAL_REPORTING_PROJECT - + + # AlloyDB PostgreSQL specific environment variables export ALLOYDB_POSTGRES_PROJECT=$PROJECT_ID export ALLOYDB_POSTGRES_REGION=$_ALLOYDB_REGION export ALLOYDB_POSTGRES_CLUSTER=$_ALLOYDB_CLUSTER diff --git a/evals/run_config.yaml b/evals/run_config.yaml index 1f2de04..d17326b 100644 --- a/evals/run_config.yaml +++ b/evals/run_config.yaml @@ -25,12 +25,16 @@ scorers: model_config: /workspace/evals/gemini_2.5_pro_model.yaml behavioral_metrics: model_config: /workspace/evals/gemini_2.5_pro_model.yaml + skills_best_practices: + model_config: /workspace/evals/gemini_2.5_pro_model.yaml + skills_dir: /workspace/alloydb/skills # Performance turn_count: {} end_to_end_latency: {} tool_call_latency: {} token_consumption: {} + skills_trajectory: {} reporting: bigquery: From 95ae9c18d86068d6bdf69abb79118e28d90c57d1 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 05:49:54 +0000 Subject: [PATCH 02/13] ci: update label check in cloudbuild.yaml to use jq exact matching instead of string parsing --- cloudbuild.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6366891..f62b3e5 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -42,11 +42,10 @@ steps: PR_DATA=$(cat pr_data.json) # Extract labels and title from PR data (Use $$ to escape bash variables) - PR_LABELS=$(echo "$$PR_DATA" | jq -r '[.labels[].name] | join(",")') PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') - # Check if execution labels are present - if [[ "$$PR_LABELS" != *"autorelease: pending"* && "$$PR_LABELS" != *"ci:run-evals"* ]]; then + # Check if execution labels are present using exact matching via jq + if ! echo "$$PR_DATA" | jq -e '.labels | any(.name == "autorelease: pending" or .name == "ci:run-evals")' > /dev/null; then echo "PR does not have 'autorelease: pending' or 'ci:run-evals' label. Skipping execution." exit 0 fi From 206d99bb3d8dd2050b2ad6e3f1f33e70ef7bbf01 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 05:52:20 +0000 Subject: [PATCH 03/13] chore: remove unused label extraction comment from cloudbuild config --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index f62b3e5..bdac0c6 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -41,7 +41,7 @@ steps: PR_DATA=$(cat pr_data.json) - # Extract labels and title from PR data (Use $$ to escape bash variables) + # Extract title from PR data (Use $$ to escape bash variables) PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') # Check if execution labels are present using exact matching via jq From 7bc2daf70c7f8ec63cab3ad6311e503c6383dc8f Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 05:54:22 +0000 Subject: [PATCH 04/13] ci: update label check in cloudbuild.yaml to read from pr_data.json instead of piped input --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index bdac0c6..432ec4f 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -45,7 +45,7 @@ steps: PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title') # Check if execution labels are present using exact matching via jq - if ! echo "$$PR_DATA" | jq -e '.labels | any(.name == "autorelease: pending" or .name == "ci:run-evals")' > /dev/null; then + if ! jq -e '.labels | any(.name == "autorelease: pending" or .name == "ci:run-evals")' pr_data.json > /dev/null; then echo "PR does not have 'autorelease: pending' or 'ci:run-evals' label. Skipping execution." exit 0 fi From 84f6f9b6fadf6f16b19eb117eaeb160f8b6edc61 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 09:27:59 +0000 Subject: [PATCH 05/13] chore: update GOOGLE_CLOUD_LOCATION to us-central1 in model configuration --- evals/model_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/model_config.yaml b/evals/model_config.yaml index 3e80dcd..f15bed6 100644 --- a/evals/model_config.yaml +++ b/evals/model_config.yaml @@ -16,7 +16,7 @@ gemini_cli_version: "@google/gemini-cli@latest" generator: gemini_cli env: GOOGLE_CLOUD_PROJECT: "${GOOGLE_CLOUD_PROJECT}" - GOOGLE_CLOUD_LOCATION: "global" + GOOGLE_CLOUD_LOCATION: "us-central1" GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_CLI_TRUST_WORKSPACE: "true" setup: From 6272ec42d926ae4b30782739e1ffc7f80b568d98 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 09:32:48 +0000 Subject: [PATCH 06/13] ci: add GEMINI_MODEL configuration set to gemini-2.5-flash --- evals/model_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/model_config.yaml b/evals/model_config.yaml index f15bed6..1d932aa 100644 --- a/evals/model_config.yaml +++ b/evals/model_config.yaml @@ -19,6 +19,7 @@ env: GOOGLE_CLOUD_LOCATION: "us-central1" GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_CLI_TRUST_WORKSPACE: "true" + GEMINI_MODEL: "gemini-2.5-flash" setup: extensions: "/workspace/alloydb": From 1cd9f06aba4622bffb9d2f68708c967031ab07c9 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 17:36:11 +0000 Subject: [PATCH 07/13] ci: update model configuration environment variables --- cloudbuild.yaml | 1 + evals/model_config.yaml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 432ec4f..85e7263 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -70,6 +70,7 @@ steps: export EVAL_GCP_PROJECT_ID=$PROJECT_ID export EVAL_GCP_PROJECT_REGION=$_ALLOYDB_REGION export EVAL_REPORTING_PROJECT=$_EVAL_REPORTING_PROJECT + export GOOGLE_CLOUD_PROJECT=$PROJECT_ID # AlloyDB PostgreSQL specific environment variables export ALLOYDB_POSTGRES_PROJECT=$PROJECT_ID diff --git a/evals/model_config.yaml b/evals/model_config.yaml index 1d932aa..2a930e6 100644 --- a/evals/model_config.yaml +++ b/evals/model_config.yaml @@ -15,11 +15,11 @@ gemini_cli_version: "@google/gemini-cli@latest" generator: gemini_cli env: - GOOGLE_CLOUD_PROJECT: "${GOOGLE_CLOUD_PROJECT}" - GOOGLE_CLOUD_LOCATION: "us-central1" + GOOGLE_CLOUD_PROJECT: "${EVAL_GCP_PROJECT_ID}" + GOOGLE_CLOUD_LOCATION: "${EVAL_GCP_PROJECT_REGION}" GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_CLI_TRUST_WORKSPACE: "true" - GEMINI_MODEL: "gemini-2.5-flash" + setup: extensions: "/workspace/alloydb": From 2f6e5b48d2cf4040d171b1cbbaed3c958a4b15ee Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 17:45:56 +0000 Subject: [PATCH 08/13] feat: update model configuration environment variables --- evals/model_config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evals/model_config.yaml b/evals/model_config.yaml index 2a930e6..4dc88a0 100644 --- a/evals/model_config.yaml +++ b/evals/model_config.yaml @@ -16,7 +16,7 @@ gemini_cli_version: "@google/gemini-cli@latest" generator: gemini_cli env: GOOGLE_CLOUD_PROJECT: "${EVAL_GCP_PROJECT_ID}" - GOOGLE_CLOUD_LOCATION: "${EVAL_GCP_PROJECT_REGION}" + GOOGLE_CLOUD_LOCATION: "global" GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_CLI_TRUST_WORKSPACE: "true" @@ -32,3 +32,4 @@ setup: ALLOYDB_POSTGRES_USER: "${ALLOYDB_POSTGRES_USER}" ALLOYDB_POSTGRES_PASSWORD: '${ALLOYDB_POSTGRES_PASSWORD}' ALLOYDB_POSTGRES_IP_TYPE: "${ALLOYDB_POSTGRES_IP_TYPE}" + GOOGLE_CLOUD_LOCATION: "${ALLOYDB_POSTGRES_REGION}" From 1859b48efb98280c22ebe0635d16a89f4eadf3ae Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 17:50:31 +0000 Subject: [PATCH 09/13] ci: update model configuration environment variables --- evals/model_config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/evals/model_config.yaml b/evals/model_config.yaml index 4dc88a0..14ba1c6 100644 --- a/evals/model_config.yaml +++ b/evals/model_config.yaml @@ -15,8 +15,10 @@ gemini_cli_version: "@google/gemini-cli@latest" generator: gemini_cli env: - GOOGLE_CLOUD_PROJECT: "${EVAL_GCP_PROJECT_ID}" + GOOGLE_CLOUD_PROJECT: "${GOOGLE_CLOUD_PROJECT}" GOOGLE_CLOUD_LOCATION: "global" + GOOGLE_CLOUD_REGION: "${EVAL_GCP_PROJECT_REGION}" + CLOUDSDK_COMPUTE_REGION: "${EVAL_GCP_PROJECT_REGION}" GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_CLI_TRUST_WORKSPACE: "true" @@ -32,4 +34,3 @@ setup: ALLOYDB_POSTGRES_USER: "${ALLOYDB_POSTGRES_USER}" ALLOYDB_POSTGRES_PASSWORD: '${ALLOYDB_POSTGRES_PASSWORD}' ALLOYDB_POSTGRES_IP_TYPE: "${ALLOYDB_POSTGRES_IP_TYPE}" - GOOGLE_CLOUD_LOCATION: "${ALLOYDB_POSTGRES_REGION}" From 754ebf3c6282c2ff5a49f7ddaee1161bc1dbb185 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Tue, 5 May 2026 18:56:56 +0000 Subject: [PATCH 10/13] ci: update evaluation prompt --- evals/dataset.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/dataset.json b/evals/dataset.json index 790ca10..8aa8ef1 100644 --- a/evals/dataset.json +++ b/evals/dataset.json @@ -2,7 +2,7 @@ "scenarios": [ { "id": "alloydb-debug-instance", - "starting_prompt": "Check my AlloyDB databases in project ${EVAL_GCP_PROJECT_ID}.", + "starting_prompt": "Check my AlloyDB clusters and instances in project ${EVAL_GCP_PROJECT_ID} to verify their availability.", "conversation_plan": "Ask the agent to list all AlloyDB instances in the project. Once all instances are listed, if ${ALLOYDB_POSTGRES_INSTANCE} exists, get its details and validate it is RUNNABLE.", "expected_trajectory": [ "list_instances", From ea34f29d1ff7ef95e6e7bca6c9def47c8b271d67 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Wed, 6 May 2026 05:46:20 +0000 Subject: [PATCH 11/13] ci: add pool connection --- cloudbuild.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 85e7263..e420ed6 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -14,6 +14,8 @@ options: logging: CLOUD_LOGGING_ONLY + pool: + name: 'projects/ext-test-alloydb/locations/us-central1/workerPools/alloydb-eval-pool' steps: From da804b24dabd7452e3cbd0c8ff29bed5ed802954 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Wed, 6 May 2026 05:50:42 +0000 Subject: [PATCH 12/13] ci: typo --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index e420ed6..4f2fcad 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -15,7 +15,7 @@ options: logging: CLOUD_LOGGING_ONLY pool: - name: 'projects/ext-test-alloydb/locations/us-central1/workerPools/alloydb-eval-pool' + name: 'projects/ext-test-alloydb/locations/us-central1/workerPools/alloydb-evals-pool' steps: From 655a280fd1a05777f146a9f1ef685369535a55ae Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 8 May 2026 05:09:29 +0000 Subject: [PATCH 13/13] ci: parameterize worker pool path in cloudbuild.yaml using project and region variables --- cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 4f2fcad..5c0c015 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -15,7 +15,7 @@ options: logging: CLOUD_LOGGING_ONLY pool: - name: 'projects/ext-test-alloydb/locations/us-central1/workerPools/alloydb-evals-pool' + name: 'projects/$PROJECT_ID/locations/$_ALLOYDB_REGION/workerPools/alloydb-evals-pool' steps: