skill with descriptive nl-sql generation

.ci/cloudbuild.yaml

@@ -101,6 +101,16 @@ steps:
       - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION'
     args: ['.ci/run_gemini_cli.sh', 'freeform-input']
 
+  - id: eval-gemini-cli-eval-dataset-gen
+    name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}'
+    entrypoint: 'bash'
+    secretEnv: ['ADC_KEY']
+    waitFor: ['build-extension', 'preflight']
+    env:
+      - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID'
+      - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION'
+    args: ['.ci/run_gemini_cli.sh', 'eval-dataset-gen']
+
   # --- Claude Code eval steps (parallel, isolated per suite) ---
   - id: eval-claude-code-core-cujs
     name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}'
@@ -124,6 +134,17 @@ steps:
       - 'CLAUDE_GCP_VERTEX_PROJECT_ID=$_CLAUDE_GCP_VERTEX_PROJECT_ID'
     args: ['.ci/run_claude_code.sh', 'freeform-input']
 
+  - id: eval-claude-code-eval-dataset-gen
+    name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}'
+    entrypoint: 'bash'
+    secretEnv: ['ADC_KEY']
+    waitFor: ['build-extension', 'preflight']
+    env:
+      - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID'
+      - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION'
+      - 'CLAUDE_GCP_VERTEX_PROJECT_ID=$_CLAUDE_GCP_VERTEX_PROJECT_ID'
+    args: ['.ci/run_claude_code.sh', 'eval-dataset-gen']
+
   # --- Upload Results and Workspaces (after all eval steps) ---
   - id: upload
     name: 'gcr.io/cloud-builders/gsutil'
@@ -133,13 +154,15 @@ steps:
       - 'eval-claude-code-smoke-test'
       - 'eval-gemini-cli-core-cujs'
       - 'eval-gemini-cli-freeform-input'
+      - 'eval-gemini-cli-eval-dataset-gen'
       - 'eval-claude-code-core-cujs'
       - 'eval-claude-code-freeform-input'
+      - 'eval-claude-code-eval-dataset-gen'
     args:
       - '-c'
       - |
         ANY_RAN=0
-        for RUN in gemini-cli-smoke-test claude-code-smoke-test gemini-cli-core-cujs gemini-cli-freeform-input claude-code-core-cujs claude-code-freeform-input; do
+        for RUN in gemini-cli-smoke-test claude-code-smoke-test gemini-cli-core-cujs gemini-cli-freeform-input gemini-cli-eval-dataset-gen claude-code-core-cujs claude-code-freeform-input claude-code-eval-dataset-gen; do
           if [ ! -f /workspace/EVAL_RAN_$$RUN ]; then
             echo "$$RUN did not run; skipping its upload."
             continue
@@ -181,7 +204,7 @@ steps:
       - |
         THRESHOLD=60
         FAILED=0
-        for RUN in gemini-cli-smoke-test claude-code-smoke-test gemini-cli-core-cujs gemini-cli-freeform-input claude-code-core-cujs claude-code-freeform-input; do
+        for RUN in gemini-cli-smoke-test claude-code-smoke-test gemini-cli-core-cujs gemini-cli-freeform-input gemini-cli-eval-dataset-gen claude-code-core-cujs claude-code-freeform-input claude-code-eval-dataset-gen; do
           if [ ! -f /workspace/EVAL_RAN_$$RUN ]; then
             continue
           fi

evals/eval-dataset-gen/README.md

@@ -0,0 +1,49 @@
+
+## CUJ: case_flights
+
+**Setup**: In this CUJ, the user only provides the https://docs.cloud.google.com/alloydb/docs/ai/integrate-applications-data-agent documentation link. 
+
+**Source**: database is `db_dummy`, which does not contain any table.
+
+**Expected Outcome**: The eval dataset generator needs to figure out based on the documentation at https://docs.cloud.google.com/alloydb/docs/ai/integrate-applications-data-agent what NL-SQL pairs to generate.
+
+
+## CUJ: case_property_search
+
+**Setup**: In this CUJ, the user only provides the github link https://github.com/kupp0/multi-db-property-search-data-agents.
+
+**Source**: database is `search`, which contains test data for the app.
+
+**Expected Outcome**: The eval dataset generator needs to figure out based on the application source codes at https://github.com/kupp0/multi-db-property-search-data-agents what NL-SQL pairs to generate.
+
+## CUJ: case_blog
+
+**Setup**: In this CUJ, the user provides the `app_data` folder containing a `ER_Diagram.jpg` which sketch the ER diagram for blog application as well as its `design_doc.pdf`. 
+
+**Source**: database is `db_dummy`, which does not contain any table.
+
+**Expected Outcome**: The eval dataset generator need to figure out that the `<source>-list-schemas` returns the empty schema (since the user did not create any table in the schema), infers the schema from the `ER_Diagram.jpg` and makes use of the content in the `design_doc.pdf` to generate NL-SQL pairs.
+
+## CUJ: case_ecommerce_cryptic
+
+**Setup**: In this CUJ, the user provides a business context doc containing explaination over the cryptic column names in the database
+
+**Source**: database is `db_ecommerce_cryptic`, which contains cryptic column names
+
+**Expected Outcome**: The eval dataset generator needs to figure out how to generate NL which leverage domain knowledge from the business context doc which explains the cryptic column names, and its corresponding SQL only knows the cryptic column names.
+
+## CUJ: case_financials
+
+**Setup**: In this CUJ, the user provides the querylog file `querylog.txt` which simulates the query log for the `financials` database.
+
+**Source**: database is `financials` from BIRD.
+
+**Expected Outcome**: The eval dataset generator needs to figure out how to extract the SQL from the query log, and then translate them to the corresponding NL while ensuring the NL is unambiguous and logically align with the SQL. In the case the number of pairs user ask is more than the SQLs in the querylog, the eval dataset generator needs to uses its knowledge and inspection of querylog to come up with more NL-SQL pairs.
+
+## CUJ: case_hr
+
+**Setup**: In this CUJ, the user provides the `app_data` directory containing the application codes.
+
+**Source**: database is the `db_hr`.
+
+**Expected Outcome**: The eval dataset generator needs to figure out which tables and columns to use to generate NL-SQL so that it alighns with the application codes.

evals/eval-dataset-gen/dataset.json

@@ -0,0 +1,49 @@
+{
+  "scenarios": [
+    {
+      "id": "autoctx:generate-dataset-resolve-biz-rule-shift",
+      "starting_prompt": "generate a evaluation dataset of SQL/Question pairs for petstore application named golden.json",
+      "conversation_plan": "Ask the agent to generate a evaluation dataset of SQL/Question pairs in postgres dialect for the petstore application whose design doc and source codes can be found in the directory `./app_data`, save the generated pairs in output file named golden.json. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
+      "expected_trajectory": [],
+      "kind": "agents",
+      "work_dir": "workspace_resolve_biz_rule_shift/",
+      "max_turns": 15
+    },
+    {
+      "id": "autoctx:generate-dataset-resolve-cryptic-schema",
+      "starting_prompt": "generate a seed evaluation dataset of SQL/Question pairs for ecommerce application named golden.json",
+      "conversation_plan": "Ask the agent to generate a evaluation dataset of SQL/Question pairs in postgres dialect for the ecommerce application whose business context doc can be found in the directory `./app_data`, save the generated pairs in output file named golden.json. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
+      "expected_trajectory": [],
+      "kind": "agents",
+      "work_dir": "workspace_resolve_cryptic_schema/",
+      "max_turns": 15
+    },
+    {
+      "id": "autoctx:generate-dataset-grounding-local-code",
+      "starting_prompt": "generate a seed evaluation dataset of SQL/Question pairs for hr application named golden.json",
+      "conversation_plan": "Ask the agent to generate a evaluation dataset of SQL/Question pairs in postgres dialect for the hr application whose source codes can be found in the directory `./app_data`, save the generated pairs in output file named golden.json. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
+      "expected_trajectory": [],
+      "kind": "agents",
+      "work_dir": "workspace_grounding_local_code/",
+      "max_turns": 15
+    },
+    {
+      "id": "autoctx:generate-dataset-grounding-github-code",
+      "starting_prompt": "generate a seed evaluation dataset of SQL/Question pairs for property search application named golden.json",
+      "conversation_plan": "Ask the agent to generate a evaluation dataset of SQL/Question pairs in postgres dialect for the application at https://github.com/kupp0/google-dach-summit26-database-labs/tree/main/labs/03_fullstack_ai_app_property_search, save the generated pairs in output file named golden.json. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
+      "expected_trajectory": [],
+      "kind": "agents",
+      "work_dir": "workspace_grounding_github_code/",
+      "max_turns": 15
+    },
+    {
+      "id": "autoctx:generate-dataset-grounding-querylog",
+      "starting_prompt": "generate a seed evaluation dataset consisting of SQL/Question pairs for my alloydb database financial named golden.json",
+      "conversation_plan": "Ask the agent to generate a evaluation dataset consisting of 5 SQL/Question pairs in postgres dialect for my alloydb database financial whose query log can be found in the file `./app_data/querylog.txt`, save the generated pairs in output file named golden.json. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.",
+      "expected_trajectory": [],
+      "kind": "agents",
+      "work_dir": "workspace_grounding_querylog/",
+      "max_turns": 15
+    }
+  ]
+}

evals/eval-dataset-gen/run_claude.yaml

@@ -0,0 +1,37 @@
+############################################################
+### Dataset / Eval Items
+############################################################
+dataset_config: eval-dataset-gen/dataset.json
+dataset_format: gemini-cli-format
+
+# Orchestrator Configuration
+# `agent` is the alias of `geminicli` that also drives Claude Code.
+orchestrator: agent
+model_config: model_configs/claude_code_model.yaml
+simulated_user_model_config: model_configs/gemini_model.yaml
+
+# Run scenarios sequentially: Claude Code shares ~/.claude session store
+# inside the sandboxed fake_home, so concurrent runs can conflict.
+runners:
+  agent_runners: 1
+
+############################################################
+### Scorer Related Configs
+############################################################
+scorers:
+  trajectory_matcher: {}
+
+  goal_completion:
+    model_config: model_configs/gemini_model.yaml
+
+  turn_count: {}
+  end_to_end_latency: {}
+  tool_call_latency: {}
+  token_consumption: {}
+
+############################################################
+### Reporting Related Configs
+############################################################
+reporting:
+  csv:
+    output_directory: 'results'

evals/eval-dataset-gen/run_gemini_cli.yaml

@@ -0,0 +1,34 @@
+############################################################
+### Dataset / Eval Items
+############################################################
+dataset_config: eval-dataset-gen/dataset.json
+dataset_format: gemini-cli-format
+
+# Orchestrator Configuration
+orchestrator: geminicli
+model_config: model_configs/gemini_cli_model.yaml
+simulated_user_model_config: model_configs/gemini_model.yaml
+
+############################################################
+### Scorer Related Configs
+############################################################
+scorers:
+  # Checks if the agent used the expected tools in the correct order.
+  trajectory_matcher: {}
+
+  # Uses an LLM to judge if the user's goal was met.
+  goal_completion:
+    model_config: model_configs/gemini_model.yaml
+
+  # Metrics for efficiency and performance.
+  turn_count: {}
+  end_to_end_latency: {}
+  tool_call_latency: {}
+  token_consumption: {}
+
+############################################################
+### Reporting Related Configs
+############################################################
+reporting:
+  csv:
+    output_directory: 'results'

evals/eval-dataset-gen/workspace_grounding_github_code/app_data/db_schema.sql

@@ -0,0 +1,61 @@
+CREATE EXTENSION IF NOT EXISTS vector CASCADE;
+CREATE EXTENSION IF NOT EXISTS alloydb_scann CASCADE;
+
+-- 2. TABLE CREATION
+DROP TABLE IF EXISTS user_prompt_history CASCADE;
+
+CREATE TABLE public.user_prompt_history (
+    id SERIAL PRIMARY KEY,
+    "timestamp" timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
+    user_prompt text,
+    prompt_embedded vector(3072),
+    query_template_used boolean,
+    query_template_id integer,
+    query_explanation text
+);
+
+DROP TABLE IF EXISTS property_listings CASCADE;
+
+CREATE TABLE property_listings (
+    id SERIAL PRIMARY KEY,
+    title VARCHAR(255) NOT NULL,
+    description TEXT,
+    price DECIMAL(12, 2) NOT NULL,
+    bedrooms INT,
+    city VARCHAR(100),
+    image_gcs_uri TEXT,
+    country VARCHAR(100) DEFAULT 'Switzerland',
+    canton VARCHAR(100),
+    -- Embeddings are generated externally and inserted directly
+    description_embedding VECTOR(3072) ,
+    image_embedding VECTOR(1408) 
+);
+
+-- 2.1 COLUMN METADATA COMMENTS (Gemini Context Enrichment)
+COMMENT ON COLUMN property_listings.bedrooms IS '<gemini>Examples: [''4'', ''6'', ''3''] | Distinct Values: 7 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.canton IS '<gemini>Examples: [''Solothurn'', ''Ticino'', ''Zug''] | Distinct Values: 27 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.city IS '<gemini>Examples: [''Stans'', ''Altdorf'', ''Kilchberg''] | Distinct Values: 89 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.country IS '<gemini>Examples: [''Switzerland''] | Distinct Values: 1 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.description IS '<gemini>Examples: [''The central rail crossroad of Switzerland. Reach anywhere fast. Modern functional apartment.'', ''Cozy retreat for weekend getaways or permanent living.''] | Distinct Values: 250 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.id IS '<gemini>Examples: [''75'', ''247'', ''13''] | Distinct Values: 250 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.image_gcs_uri IS '<gemini>Examples: [''https://storage.googleapis.com/property-images-data-agent-ai-powered-search-alloydb-1542/listings/10.jpg''] | Distinct Values: 250 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.price IS '<gemini>Examples: [''11878.00'', ''4869.00'', ''2792.00''] | Distinct Values: 189 | Null Count: 0 |</gemini>';
+COMMENT ON COLUMN property_listings.title IS '<gemini>Examples: [''Rustic Studio in Landquart'', ''Renovated Villa in Herisau'', ''Quiet Home in Appenzell''] | Distinct Values: 248 | Null Count: 0 |</gemini>';
+
+
+-- 3. INDEX CREATION (ScaNN). Follow index sql files after data is loaded. 
+
+-- 4. MODEL ALIASING (Vertex AI Integration)
+CREATE EXTENSION IF NOT EXISTS google_ml_integration CASCADE;
+
+
+-- 4.1 Test Text Embeddings in Database Vertex AI integration
+SELECT google_ml.embedding(
+    model_id => 'gemini-embedding-001',
+    content => 'This is the text to embed.'
+);
+SELECT google_ml.embedding(
+    model_id => 'multimodalembedding@001',
+    content => 'This is the text to embed.'
+);
+

evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/state.md

@@ -0,0 +1 @@
+# context authoring experiment state tracking

evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/tools.yaml

@@ -0,0 +1,33 @@
+kind: source
+name: my-alloydb
+type: alloydb-postgres
+project: cloud-db-nl2sql
+region: us-central1
+cluster: whaoyu-test
+instance: whaoyu-test-primary
+database: search
+---
+kind: tool
+name: my-alloydb-list-schemas
+type: postgres-list-tables
+source: my-alloydb
+description: |
+  Use this tool to list tables and their schemas in the my-alloydb database.
+
+  Progressive Schema Discovery (Recommended):
+  1) Fetch structure first (output_format='simple'),
+  2) Go deep on specific parts if interested,
+  3) Use batching if info is too large.
+
+  Scope:
+  - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data.
+
+  Behavior:
+  - Omit 'table_names' to fetch all tables.
+  - Omit 'output_format' for detailed schema (default).
+---
+kind: tool
+name: my-alloydb-execute-sql
+type: postgres-execute-sql
+source: my-alloydb
+description: Use this tool to execute SQL statements against the my-alloydb database.