feat: Initial development of seed eval dataset generation

.ci/cloudbuild.yaml

@@ -1,6 +1,7 @@
 steps:
   # --- Build Extension Step ---
-  - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
+  - id: build-extension
+    name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
     entrypoint: 'bash'
     args:
       - '-c'
@@ -10,77 +11,97 @@ steps:
         chmod +x .ci/build.sh
         .ci/build.sh
 
-  # --- Evaluation Step ---
-  - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
+   # --- Pre-flight: gate evals on PR label ---
+  # Runs in parallel with build-extension; writes /workspace/SHOULD_RUN
+  # if evals should proceed (always for manual runs; only on 'ci:eval' label
+  # for PR-triggered runs).
+  - id: preflight
+    name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
     entrypoint: 'bash'
-    secretEnv: ['ADC_KEY', 'GITHUB_TOKEN']
+    secretEnv: ['GITHUB_TOKEN']
+    waitFor: ['-']
     args:
       - '-c'
       - |
         set -e
 
         if [ -z "$_PR_NUMBER" ]; then
-          echo "No PR number provided. Skipping PR check (assuming local/manual run)."
-        else
-          echo "Fetching PR data from GitHub API..."
-
-          # Fetch PR data and status code
-          HTTP_STATUS=$(curl -s -o pr_data.json -w "%{http_code}" -H "Authorization: token $$GITHUB_TOKEN" \
-            "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER")
-
-          if [ "$$HTTP_STATUS" -ne 200 ]; then
-            echo "Error fetching PR data: HTTP $$HTTP_STATUS"
-            cat pr_data.json
-            exit 1
-          fi
-
-          # Check if execution labels are present
-          if ! jq -e '.labels | any(.name == "ci:eval")' pr_data.json > /dev/null; then
-            echo "PR does not have 'ci:eval' label. Skipping execution."
-            exit 0
-          fi
-          echo "Execution label detected. Processing release version context..."
+          echo "No PR number provided. Assuming local/manual run — enabling evals."
+          touch /workspace/SHOULD_RUN
+          exit 0
         fi
 
-        # Write secret to file for ADC
-        echo "$$ADC_KEY" > /tmp/adc.json
-        chmod 600 /tmp/adc.json
-        export GOOGLE_APPLICATION_CREDENTIALS=/tmp/adc.json
-
-        cd /workspace/evals
-
-        # Override model.yaml to use the locally built extension instead of GitHub
-        sed -i 's|https://github.com/GoogleCloudPlatform/db-context-enrichment|/workspace/staging|g' core-cujs/model.yaml
-
-        # Set environment variables for evalbench
-        export EVAL_GCP_PROJECT_ID=$PROJECT_ID
-        export EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION
-        export GOOGLE_CLOUD_PROJECT=$PROJECT_ID
+        echo "Fetching PR data from GitHub API..."
+        HTTP_STATUS=$(curl -s -o pr_data.json -w "%{http_code}" -H "Authorization: token $$GITHUB_TOKEN" \
+          "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER")
 
-        # Setup Python path and run evalbench
-        export PYTHONPATH=/evalbench:/evalbench/evalproto
-        export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+        if [ "$$HTTP_STATUS" -ne 200 ]; then
+          echo "Error fetching PR data: HTTP $$HTTP_STATUS"
+          cat pr_data.json
+          exit 1
+        fi
 
-        echo "Launching Standalone Evaluation..."
-        python3 /evalbench/evalbench/evalbench.py --experiment_config=core-cujs/run.yaml
+        if ! jq -e '.labels | any(.name == "ci:eval")' pr_data.json > /dev/null; then
+          echo "PR does not have 'ci:eval' label. Skipping evals."
+          exit 0
+        fi
 
-        # Mark that evaluation ran
-        touch /workspace/EVAL_RAN
+        echo "ci:eval label detected — enabling evals."
+        touch /workspace/SHOULD_RUN
 
-  # --- Upload Results and Workspaces ---
-  - name: 'gcr.io/cloud-builders/gsutil'
+  # --- Eval steps (parallel, isolated per suite) ---
+  # Both steps run the same .ci/run_eval.sh script with a different suite arg.
+  - id: eval-core-cujs
+    name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
+    entrypoint: 'bash'
+    secretEnv: ['ADC_KEY']
+    waitFor: ['build-extension', 'preflight']
+    env:
+      - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID'
+      - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION'
+    args: ['.ci/run_eval.sh', 'core-cujs']
+
+  - id: eval-seed-eval-gen
+    name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
+    entrypoint: 'bash'
+    secretEnv: ['ADC_KEY']
+    waitFor: ['build-extension', 'preflight']
+    env:
+      - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID'
+      - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION'
+    args: ['.ci/run_eval.sh', 'seed-eval-gen']
+
+  # --- Upload Results and Workspaces (after both eval steps) ---
+  - id: upload
+    name: 'gcr.io/cloud-builders/gsutil'
     entrypoint: 'bash'
+    waitFor: ['eval-core-cujs', 'eval-seed-eval-gen']
     args:
       - '-c'
       - |
-        if [ -f /workspace/EVAL_RAN ]; then
-          echo "Uploading results..."
-          gsutil -m cp -r evals/* gs://context-engineering-agent-eval-results/evals/$BUILD_ID/
+        ANY_RAN=0
+        for SUITE in core-cujs seed-eval-gen; do
+          if [ ! -f /workspace/EVAL_RAN_$$SUITE ]; then
+            echo "$$SUITE did not run; skipping its upload."
+            continue
+          fi
+          ANY_RAN=1
+          WORK_DIR=/workspace/eval-$$SUITE
+          DEST=gs://context-engineering-agent-eval-results/evals/$BUILD_ID/$$SUITE
+
+          echo "Uploading $$SUITE results to $$DEST ..."
+          gsutil -m cp -r $$WORK_DIR/* $$DEST/
+
+          if [ -d $$WORK_DIR/.venv/fake_home/.gemini/tmp ]; then
+            echo "Uploading $$SUITE Gemini workspace..."
+            gsutil -m cp -r $$WORK_DIR/.venv/fake_home/.gemini/tmp/* $$DEST/gemini/
+          fi
+        done
 
-          echo "Uploading Gemini workspace..."
-          gsutil -m cp -r evals/.venv/fake_home/.gemini/tmp/* gs://context-engineering-agent-eval-results/evals/$BUILD_ID/gemini/
+        if [ "$$ANY_RAN" = "1" ]; then
+          echo "View uploaded results at: https://console.cloud.google.com/storage/browser/context-engineering-agent-eval-results/evals/$BUILD_ID"
         else
-          echo "Evaluation did not run. Skipping upload."
+          echo "No evaluations ran. Skipping upload."
         fi
 
 availableSecrets:

.ci/run_eval.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# Run a single evalbench suite in an isolated working directory.
+#
+# Invoked from .ci/cloudbuild.yaml. Required env vars: ADC_KEY,
+# GEMINI_API_KEY, EVAL_GCP_PROJECT_ID, EVAL_GCP_PROJECT_REGION.
+# Required argument: <suite-name> (e.g. core-cujs, freeform-input).
+
+set -e
+
+SUITE="${1:?usage: run_eval.sh <suite-name>}"
+
+if [ ! -f /workspace/SHOULD_RUN ]; then
+  echo "Evals disabled by preflight; skipping ${SUITE}."
+  exit 0
+fi
+
+# ADC setup
+echo "${ADC_KEY}" > /tmp/adc.json
+chmod 600 /tmp/adc.json
+export GOOGLE_APPLICATION_CREDENTIALS=/tmp/adc.json
+
+# Isolate: copy this suite into its own working directory so parallel
+# runs don't fight over the same .venv / fake_home / .gemini/tmp state.
+WORK_DIR="/workspace/eval-${SUITE}"
+mkdir -p "${WORK_DIR}"
+cp -r "/workspace/evals/${SUITE}" "${WORK_DIR}/"
+cd "${WORK_DIR}"
+
+# Point model.yaml at the locally built extension and inject the Gemini API
+# key into the CLI's env (orchestrator auth — the extension itself no longer
+# needs the key, so we no longer touch the `settings: {}` block).
+sed -i 's|https://github.com/GoogleCloudPlatform/db-context-enrichment|/workspace/staging|g' "${SUITE}/model.yaml"
+
+# Inject Vertex project/location into the CLI env block so the extension can
+# talk to the right GCP project without the values being committed to the repo.
+sed -i \
+  -e "/^  GEMINI_MODEL:/a\\  GOOGLE_CLOUD_PROJECT: \"${EVAL_GCP_PROJECT_ID}\"" \
+  -e "/^  GEMINI_MODEL:/a\\  GOOGLE_CLOUD_LOCATION: \"${EVAL_GCP_PROJECT_REGION}\"" \
+  "${SUITE}/model.yaml"
+
+# evalbench runtime
+export PYTHONPATH=/evalbench:/evalbench/evalproto
+export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
+
+echo "Launching ${SUITE} evaluation..."
+uv run --no-sync --project /evalbench python /evalbench/evalbench/evalbench.py --experiment_config="${SUITE}/run.yaml"
+
+touch "/workspace/EVAL_RAN_${SUITE}"

.python-version

@@ -1 +1 @@
-3.12
+3.12.13

evals/core-cujs/dataset.json

@@ -9,14 +9,23 @@
       "max_turns": 15
     },
     {
-      "id": "autoctx:generate-dataset",
+      "id": "autoctx:expand-dataset",
       "starting_prompt": "generate a evaluation dataset for my alloydb database financial named golden.json, based on my seed dataset to be provided.",
       "conversation_plan": "Ask the agent to generate a evaluation dataset for my alloydb database financial named golden.json. The 'seed' includes the following two golden NL-SQL pair:\n\nHow many accounts who choose issuance after transaction are staying in East Bohemia region? A3 contains the data of region; 'POPLATEK PO OBRATU' represents for 'issuance after transaction'.\nSELECT COUNT(DISTINCT \"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"district\" AS \"T2\" ON \"T1\".\"district_id\" = \"T2\".\"district_id\" WHERE \"T2\".\"A3\" = 'east Bohemia' AND \"T1\".\"frequency\" = 'POPLATEK PO OBRATU'\n\nHow many accounts who have region in Prague are eligible for loans? A3 contains the data of region\nSELECT COUNT(\"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"loan\" AS \"T2\" ON \"T1\".\"account_id\" = \"T2\".\"account_id\" INNER JOIN \"district\" AS \"T3\" ON \"T1\".\"district_id\" = \"T3\".\"district_id\" WHERE \"T3\".\"A3\" = 'Prague'. If prompted for validating the query, reply yes to the agent. Once the initial golden dataset is generated, ask the agent to expand it with 2 additional examples in the same file. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
       "expected_trajectory": [],
       "kind": "agents",
       "work_dir": "workspace_post_init/",
       "max_turns": 15
     },
+    {
+      "id": "autoctx:generate-dataset",
+      "starting_prompt": "generate a evaluation dataset consisting of simple SQL/Question pairs for my alloydb database financial named golden_seed.json",
+      "conversation_plan": "Ask the agent to generate a evaluation dataset consisting of 5 simple SQL/Question pairs for my alloydb database financial named golden_seed.json. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
+      "expected_trajectory": [],
+      "kind": "agents",
+      "work_dir": "workspace_post_init/",
+      "max_turns": 15
+    },
     {
       "id": "autoctx:bootstrap",
       "starting_prompt": "create initial context set for my alloydb database",

evals/seed-eval-gen/README.md

@@ -0,0 +1,49 @@
+
+## CUJ: case_flights
+
+**Setup**: In this CUJ, the user only provides the https://docs.cloud.google.com/alloydb/docs/ai/integrate-applications-data-agent documentation link. 
+
+**Source**: database is `db_dummy`, which does not contain any table.
+
+**Expected Outcome**: The eval dataset generator needs to figure out based on the documentation at https://docs.cloud.google.com/alloydb/docs/ai/integrate-applications-data-agent what NL-SQL pairs to generate.
+
+
+## CUJ: case_property_search
+
+**Setup**: In this CUJ, the user only provides the github link https://github.com/kupp0/multi-db-property-search-data-agents.
+
+**Source**: database is `search`, which contains test data for the app.
+
+**Expected Outcome**: The eval dataset generator needs to figure out based on the application source codes at https://github.com/kupp0/multi-db-property-search-data-agents what NL-SQL pairs to generate.
+
+## CUJ: case_blog
+
+**Setup**: In this CUJ, the user provides the `app_data` folder containing a `ER_Diagram.jpg` which sketch the ER diagram for blog application as well as its `design_doc.pdf`. 
+
+**Source**: database is `db_dummy`, which does not contain any table.
+
+**Expected Outcome**: The eval dataset generator need to figure out that the `<source>-list-schemas` returns the empty schema (since the user did not create any table in the schema), infers the schema from the `ER_Diagram.jpg` and makes use of the content in the `design_doc.pdf` to generate NL-SQL pairs.
+
+## CUJ: case_ecommerce_cryptic
+
+**Setup**: In this CUJ, the user provides a business context doc containing explaination over the cryptic column names in the database
+
+**Source**: database is `db_ecommerce_cryptic`, which contains cryptic column names
+
+**Expected Outcome**: The eval dataset generator needs to figure out how to generate NL which leverage domain knowledge from the business context doc which explains the cryptic column names, and its corresponding SQL only knows the cryptic column names.
+
+## CUJ: case_financials
+
+**Setup**: In this CUJ, the user provides the querylog file `querylog.txt` which simulates the query log for the `financials` database.
+
+**Source**: database is `financials` from BIRD.
+
+**Expected Outcome**: The eval dataset generator needs to figure out how to extract the SQL from the query log, and then translate them to the corresponding NL while ensuring the NL is unambiguous and logically align with the SQL. In the case the number of pairs user ask is more than the SQLs in the querylog, the eval dataset generator needs to uses its knowledge and inspection of querylog to come up with more NL-SQL pairs.
+
+## CUJ: case_hr
+
+**Setup**: In this CUJ, the user provides the `app_data` directory containing the application codes.
+
+**Source**: database is the `db_hr`.
+
+**Expected Outcome**: The eval dataset generator needs to figure out which tables and columns to use to generate NL-SQL so that it alighns with the application codes.