From 1cf64e3a07c6da4e2f9dbe2d533f4b914be60623 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Sun, 19 Jul 2026 16:45:42 -0700 Subject: [PATCH 01/19] wip: squash of decouple + workflow rewrite + evals overhaul + evaluate coord rule --- .ci/build.sh | 2 +- .ci/cloudbuild-oneoff.yaml | 62 +++++ .ci/run_gemini_cli.sh | 11 +- .github/workflows/release.yml | 2 +- README.md | 2 +- dev-plugin/gemini-extension.json | 2 +- dev-plugin/plugin/.claude-plugin/plugin.json | 2 +- evals/core-cujs/dataset.json | 19 +- .../.context-engineering/tools.yaml | 33 +++ .../bootstrap_context.json | 0 .../workspace_fresh_hillclimb/golden.json | 26 ++ .../.context-engineering/tools.yaml | 33 +++ .../workspace_post_bootstrap/autoctx/state.md | 1 - .../bootstrap_context.json | 0 .../.context-engineering/tools.yaml | 33 +++ .../autoctx/state.md | 1 - .../my-alloydb-tuning-experiment/state.md | 16 ++ .../v0/context_set_v0.json | 44 ++++ .../v0/eval}/eval_configs/db_config.yaml | 0 .../v0/eval}/eval_configs/golden_queries.json | 0 .../eval}/eval_configs/llmrater_config.yaml | 0 .../v0/eval}/eval_configs/model_config.yaml | 0 .../v0/eval}/eval_configs/run_config.yaml | 0 .../configs.csv | 0 .../evals.csv | 0 .../scores.csv | 0 .../summary.csv | 0 .../.context-engineering}/tools.yaml | 0 .../autoctx/state.md | 9 - .../.context-engineering/tools.yaml | 33 +++ .../experiments/.gitkeep | 0 .../.context-engineering/tools.yaml | 33 +++ .../workspace_post_init/autoctx/state.md | 1 - .../experiments/.gitkeep | 0 .../.context-engineering}/tools.yaml | 2 +- .../.context-engineering/experiments/.gitkeep | 0 .../.context-engineering}/tools.yaml | 2 +- .../.context-engineering/experiments/.gitkeep | 0 .../.context-engineering}/tools.yaml | 0 .../.context-engineering/experiments/.gitkeep | 0 .../.context-engineering/tools.yaml | 33 +++ .../.context-engineering/experiments/.gitkeep | 0 .../.context-engineering/tools.yaml | 33 +++ evals/freeform-input/dataset.json | 8 +- .../.context-engineering}/tools.yaml | 0 .../.context-engineering}/tools.yaml | 0 evals/smoke-test/dataset.json | 4 +- .../.context-engineering/experiments/.gitkeep | 0 .../.context-engineering}/tools.yaml | 0 .../workspace_post_init/autoctx/state.md | 1 - gemini-extension.json | 2 +- plugin/.claude-plugin/plugin.json | 2 +- .../context-engineering-bootstrap/SKILL.md | 109 ++++----- .../context-engineering-evaluate/SKILL.md | 136 +++++------ .../SKILL.md | 5 +- .../references/facet/googlesql.md | 0 .../references/facet/mysql.md | 0 .../references/facet/postgresql.md | 0 .../phrase_extraction/guidelines.md | 0 .../references/template/googlesql.md | 0 .../references/template/mysql.md | 0 .../references/template/postgresql.md | 0 .../references/value_search/googlesql.md | 0 .../references/value_search/mysql.md | 0 .../references/value_search/postgresql.md | 0 .../context-engineering-hillclimb/SKILL.md | 223 ++++++------------ .../references/workspace.md | 85 +++++++ .../skills/context-engineering-init/SKILL.md | 142 +++++------ .../context-engineering-workflow/SKILL.md | 140 +++++------ .../evaluate/db_generators/base.py | 6 + .../cloud/db_context_enrichment/main.py | 9 +- .../evaluate/db_generators/alloydb_test.py | 1 + .../evaluate/db_generators/mysql_test.py | 1 + .../evaluate/db_generators/postgres_test.py | 1 + .../evaluate/db_generators/spanner_test.py | 1 + .../evaluate/evaluate_generator_test.py | 1 + 76 files changed, 827 insertions(+), 485 deletions(-) create mode 100644 .ci/cloudbuild-oneoff.yaml create mode 100644 evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml rename evals/core-cujs/{workspace_post_bootstrap/autoctx/experiments/my-alloydb-tuning-experiment => workspace_fresh_hillclimb}/bootstrap_context.json (100%) create mode 100644 evals/core-cujs/workspace_fresh_hillclimb/golden.json create mode 100644 evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml delete mode 100644 evals/core-cujs/workspace_post_bootstrap/autoctx/state.md rename evals/core-cujs/{workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment => workspace_post_bootstrap}/bootstrap_context.json (100%) create mode 100644 evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml delete mode 100644 evals/core-cujs/workspace_post_dataset_generation/autoctx/state.md create mode 100644 evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md create mode 100644 evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/context_set_v0.json rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_configs/db_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_configs/golden_queries.json (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_configs/llmrater_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_configs/model_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_configs/run_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv (100%) rename evals/core-cujs/workspace_post_evaluation/{autoctx/experiments/my-alloydb-tuning-experiment => .context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval}/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv (100%) rename evals/core-cujs/{workspace_post_bootstrap/autoctx => workspace_post_evaluation/.context-engineering}/tools.yaml (100%) delete mode 100644 evals/core-cujs/workspace_post_evaluation/autoctx/state.md create mode 100644 evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml rename evals/core-cujs/workspace_post_init/{autoctx => .context-engineering}/experiments/.gitkeep (100%) create mode 100644 evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml delete mode 100644 evals/core-cujs/workspace_post_init/autoctx/state.md rename evals/{smoke-test/workspace_post_init/autoctx => eval-dataset-gen/workspace_grounding_github_code/.context-engineering}/experiments/.gitkeep (100%) rename evals/{freeform-input/workspace_missing_info_exploration/autoctx => eval-dataset-gen/workspace_grounding_github_code/.context-engineering}/tools.yaml (97%) create mode 100644 evals/eval-dataset-gen/workspace_grounding_local_code/.context-engineering/experiments/.gitkeep rename evals/{smoke-test/workspace_post_init/autoctx => eval-dataset-gen/workspace_grounding_local_code/.context-engineering}/tools.yaml (97%) create mode 100644 evals/eval-dataset-gen/workspace_grounding_querylog/.context-engineering/experiments/.gitkeep rename evals/{core-cujs/workspace_post_dataset_generation/autoctx => eval-dataset-gen/workspace_grounding_querylog/.context-engineering}/tools.yaml (100%) create mode 100644 evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/experiments/.gitkeep create mode 100644 evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/tools.yaml create mode 100644 evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/experiments/.gitkeep create mode 100644 evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/tools.yaml rename evals/{core-cujs/workspace_post_evaluation/autoctx => freeform-input/workspace_fix_query_gap/.context-engineering}/tools.yaml (100%) rename evals/{core-cujs/workspace_post_init/autoctx => freeform-input/workspace_missing_info_exploration/.context-engineering}/tools.yaml (100%) create mode 100644 evals/smoke-test/workspace_post_init/.context-engineering/experiments/.gitkeep rename evals/{freeform-input/workspace_fix_query_gap/autoctx => smoke-test/workspace_post_init/.context-engineering}/tools.yaml (100%) delete mode 100644 evals/smoke-test/workspace_post_init/autoctx/state.md rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/SKILL.md (99%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/facet/googlesql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/facet/mysql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/facet/postgresql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/phrase_extraction/guidelines.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/template/googlesql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/template/mysql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/template/postgresql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/value_search/googlesql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/value_search/mysql.md (100%) rename plugin/skills/{context-generation-guide => context-engineering-generation-guide}/references/value_search/postgresql.md (100%) create mode 100644 plugin/skills/context-engineering-hillclimb/references/workspace.md diff --git a/.ci/build.sh b/.ci/build.sh index 3a28f0d1..da044d0e 100644 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -58,7 +58,7 @@ mv evalbench staging/skills/autoctx-evaluate/scripts/ BINARY_NAME="\${extensionPath}/google-cloud-db-context-engineering" TOOLBOX_NAME="\${extensionPath}/skills/autoctx-init/scripts/toolbox" -jq ".contextFileName = \"GEMINI.md\" | .mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \"autoctx/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json +jq ".contextFileName = \"GEMINI.md\" | .mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \".context-engineering/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json cp plugin/GEMINI.md staging/ cp LICENSE staging/ diff --git a/.ci/cloudbuild-oneoff.yaml b/.ci/cloudbuild-oneoff.yaml new file mode 100644 index 00000000..27916abd --- /dev/null +++ b/.ci/cloudbuild-oneoff.yaml @@ -0,0 +1,62 @@ +# One-off Cloud Build config for running a single evalbench scenario against +# the Gemini CLI SUT. Bypasses PR-label gating, preflight, and pass-rate +# checks — intended for ad-hoc developer runs. +# +# Usage: +# gcloud builds submit --config=.ci/cloudbuild-oneoff.yaml \ +# --substitutions=_SUITE=core-cujs,_SCENARIOS=autoctx-init \ +# --project=cloud-db-nl2sql +steps: + # --- Build extension (same as the main pipeline) --- + - id: build-extension + name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}' + entrypoint: 'bash' + args: + - '-c' + - | + set -e + apt-get update && apt-get install -y binutils + chmod +x .ci/build.sh + .ci/build.sh + + # --- Marker + release-version files (replacing preflight for manual runs) --- + - id: preflight-oneoff + name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}' + entrypoint: 'bash' + waitFor: ['-'] + args: + - '-c' + - | + set -e + touch /workspace/SHOULD_RUN_GEMINI_CLI + printf 'manual-oneoff-%s' "$BUILD_ID" > /workspace/RELEASE_VERSION + + # --- The scoped eval --- + - id: eval-gemini-cli-oneoff + name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}' + entrypoint: 'bash' + secretEnv: ['ADC_KEY'] + waitFor: ['build-extension', 'preflight-oneoff'] + env: + - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' + - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' + - 'EVAL_REPORTING_PROJECT=$_EVAL_REPORTING_PROJECT' + - 'EVAL_SCENARIOS=$_SCENARIOS' + args: ['.ci/run_gemini_cli.sh', '$_SUITE'] + +availableSecrets: + secretManager: + - versionName: projects/cloud-db-nl2sql/secrets/context-engineering-agent-adc-credentials/versions/latest + env: 'ADC_KEY' + +serviceAccount: 'projects/cloud-db-nl2sql/serviceAccounts/crema-eval@cloud-db-nl2sql.iam.gserviceaccount.com' + +substitutions: + _EVALBENCH_IMAGE_TAG: 'latest' + _PROJECT_LOCATION: 'global' + _EVAL_REPORTING_PROJECT: 'cloud-db-nl2sql' + _SUITE: 'core-cujs' + _SCENARIOS: 'autoctx-init' + +options: + defaultLogsBucketBehavior: REGIONAL_USER_OWNED_BUCKET diff --git a/.ci/run_gemini_cli.sh b/.ci/run_gemini_cli.sh index 65018c19..a4ea9c57 100755 --- a/.ci/run_gemini_cli.sh +++ b/.ci/run_gemini_cli.sh @@ -54,8 +54,15 @@ sed -i "s|\${EVAL_REPORTING_PROJECT}|${EVAL_REPORTING_PROJECT:-}|g" "${CONFIG}" export PYTHONPATH=/evalbench:/evalbench/evalproto export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python -echo "Launching ${SUT}/${SUITE} evaluation..." -uv run --no-sync --project /evalbench python /evalbench/evalbench/evalbench.py --experiment_config="${SUITE}/run_gemini_cli.yaml" +# Optional scenario filter: set EVAL_SCENARIOS=[,...] to run a subset. +SCENARIO_ARG="" +[ -n "${EVAL_SCENARIOS:-}" ] && SCENARIO_ARG="--scenarios=${EVAL_SCENARIOS}" + +echo "Launching ${SUT}/${SUITE} evaluation${SCENARIO_ARG:+ (scenarios: ${EVAL_SCENARIOS})}..." +uv run --no-sync --project /evalbench python /evalbench/evalbench/evalbench.py --experiment_config="${SUITE}/run_gemini_cli.yaml" ${SCENARIO_ARG} + +echo "Validating mandatory output files for ${SUITE}..." +python3 /workspace/.ci/check_eval_outputs.py "${WORK_DIR}/${SUITE}" "${WORK_DIR}/${SUITE}/dataset.json" echo "Validating mandatory output files for ${SUITE}..." python3 /workspace/.ci/check_eval_outputs.py "${WORK_DIR}/${SUITE}" "${WORK_DIR}/${SUITE}/dataset.json" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a372ce62..571ecc69 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -142,7 +142,7 @@ jobs: TOOLBOX_NAME="\${extensionPath}/skills/autoctx-init/scripts/toolbox" fi - jq ".contextFileName = \"GEMINI.md\" | .mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \"autoctx/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json + jq ".contextFileName = \"GEMINI.md\" | .mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \".context-engineering/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json cp plugin/GEMINI.md staging/ cp LICENSE staging/ diff --git a/README.md b/README.md index 2e780a59..ede73c65 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ All optimization lifecycle phases are managed and orchestrated by the single **` ### Targeted Manual Generation -These are more basic workflows for context engineering to manually author specific context elements, each exposed via the `context-generation-guide` skill scoped to the relevant context type: +These are more basic workflows for context engineering to manually author specific context elements, each exposed via the `context-engineering-generation-guide` skill scoped to the relevant context type: * **Generate Templates**: Initiates a guided workflow where you provide a sample question and SQL, and the agent helps you parameterize and save it as a template. * **Generate Facets**: Guides you to define a specific intent and the corresponding SQL snippet (e.g., filter condition) to save as a facet. diff --git a/dev-plugin/gemini-extension.json b/dev-plugin/gemini-extension.json index 701df821..23c55e6e 100644 --- a/dev-plugin/gemini-extension.json +++ b/dev-plugin/gemini-extension.json @@ -17,7 +17,7 @@ "args": [ "toolbox-server@1.4.0", "--config", - "autoctx/tools.yaml", + ".context-engineering/tools.yaml", "--stdio" ] } diff --git a/dev-plugin/plugin/.claude-plugin/plugin.json b/dev-plugin/plugin/.claude-plugin/plugin.json index caaee887..2668dabb 100644 --- a/dev-plugin/plugin/.claude-plugin/plugin.json +++ b/dev-plugin/plugin/.claude-plugin/plugin.json @@ -20,7 +20,7 @@ "args": [ "toolbox-server@1.4.0", "--config", - "autoctx/tools.yaml", + ".context-engineering/tools.yaml", "--stdio" ] } diff --git a/evals/core-cujs/dataset.json b/evals/core-cujs/dataset.json index 35460bb9..47ba4d0d 100644 --- a/evals/core-cujs/dataset.json +++ b/evals/core-cujs/dataset.json @@ -3,7 +3,7 @@ { "id": "autoctx-init", "starting_prompt": "setup context generation for my database.", - "conversation_plan": "Ask the agent to setup the auto context generation for my alloydb database. Connection information: datasource name as my-alloydb, project id is cloud-db-nl2sql, region us-central1, cluster whaoyu-test, instance whaoyu-test-primary, database financial. Verify the agent generate a valid tools.yaml. Explicitly ask the agent to perform validation. You should terminate the conversation immediately after the agent successfully set up the files and performed the validation.", + "conversation_plan": "Ask the agent to setup the auto context generation for my alloydb database. Connection information: datasource name as my-alloydb, project id is astana-transformation, region us-east4, cluster juexinw-test, instance juexinw-test-primary, database financial. Verify the agent generate a valid tools.yaml. Explicitly ask the agent to perform validation. You should terminate the conversation immediately after the agent successfully set up the files and performed the validation.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_empty/", @@ -38,21 +38,30 @@ }, { "id": "autoctx-evaluate", - "starting_prompt": "evaluate on the golden dataset golden.json with context set id projects/cloud-db-nl2sql/locations/us-east1/contextSets/whaoyu-eval-contextset", - "conversation_plan": "Ask the agent to evaluate the golden dataset golden.json, using the current context set with context set id projects/cloud-db-nl2sql/locations/us-east1/contextSets/whaoyu-eval-contextset. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully evaluated the dataset, even if the agent prompts you to perform context improving task or take next steps.", + "starting_prompt": "Evaluate golden.json against my local context set at bootstrap_context.json. Upload it with project_id=astana-transformation, csg_id=my-alloydb-tuning-experiment, cs_id=autoctx, version=v0.", + "conversation_plan": "The agent should upload the local ContextSet using the exact coordinates in the starting prompt (no substitution), run the evaluation, and report scored results. Terminate after the results are presented.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_bootstrap/", "max_turns": 5 }, { - "id": "autoctx-hillclimb", + "id": "autoctx-hillclimb-continue", "starting_prompt": "improve the context set based on eval failure", - "conversation_plan": "Ask the agent to perform the hillclimbing to improve the context set based on eval failure. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully performed the hillclimbing task, even if the agent prompts you to perform evaluation or take next steps.", + "conversation_plan": "Ask the agent to perform the hillclimbing to improve the context set based on eval failure. The workspace already contains an in-progress experiment at .context-engineering/experiments/my-alloydb-tuning-experiment/ with a v0 baseline. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully performed the hillclimbing task, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_evaluation/", "max_turns": 5 + }, + { + "id": "autoctx-hillclimb-fresh", + "starting_prompt": "improve my context set from scratch using the golden dataset", + "conversation_plan": "Ask the agent to run hillclimb using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. If prompted for experiment name, use 'my-alloydb-fresh-experiment'. If prompted for Context Store coordinates, use csg_id 'my-alloydb-fresh-experiment', cs_id 'autoctx', version 'v0'. Terminate immediately after the agent successfully performs the hillclimbing task, even if the agent prompts you to continue.", + "expected_trajectory": [], + "kind": "agents", + "work_dir": "workspace_fresh_hillclimb/", + "max_turns": 5 } ] } \ No newline at end of file diff --git a/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml b/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml new file mode 100644 index 00000000..c66f2491 --- /dev/null +++ b/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: astana-transformation +region: us-east4 +cluster: juexinw-test +instance: juexinw-test-primary +database: financial +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/core-cujs/workspace_post_bootstrap/autoctx/experiments/my-alloydb-tuning-experiment/bootstrap_context.json b/evals/core-cujs/workspace_fresh_hillclimb/bootstrap_context.json similarity index 100% rename from evals/core-cujs/workspace_post_bootstrap/autoctx/experiments/my-alloydb-tuning-experiment/bootstrap_context.json rename to evals/core-cujs/workspace_fresh_hillclimb/bootstrap_context.json diff --git a/evals/core-cujs/workspace_fresh_hillclimb/golden.json b/evals/core-cujs/workspace_fresh_hillclimb/golden.json new file mode 100644 index 00000000..e02d883a --- /dev/null +++ b/evals/core-cujs/workspace_fresh_hillclimb/golden.json @@ -0,0 +1,26 @@ +[ + { + "id": "eval_001", + "database": "financial", + "nlq": "How many accounts who choose issuance after transaction are staying in East Bohemia region? A3 contains the data of region; 'POPLATEK PO OBRATU' represents for 'issuance after transaction'.", + "golden_sql": "SELECT COUNT(DISTINCT \"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"district\" AS \"T2\" ON \"T1\".\"district_id\" = \"T2\".\"district_id\" WHERE \"T2\".\"A3\" = 'east Bohemia' AND \"T1\".\"frequency\" = 'POPLATEK PO OBRATU'" + }, + { + "id": "eval_002", + "database": "financial", + "nlq": "How many accounts who have region in Prague are eligible for loans? A3 contains the data of region", + "golden_sql": "SELECT COUNT(\"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"loan\" AS \"T2\" ON \"T1\".\"account_id\" = \"T2\".\"account_id\" INNER JOIN \"district\" AS \"T3\" ON \"T1\".\"district_id\" = \"T3\".\"district_id\" WHERE \"T3\".\"A3\" = 'Prague'" + }, + { + "id": "eval_003", + "database": "financial", + "nlq": "How many accounts who choose monthly issuance are staying in south Moravia region? 'POPLATEK MESICNE' represents for 'monthly issuance'.", + "golden_sql": "SELECT COUNT(DISTINCT \"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"district\" AS \"T2\" ON \"T1\".\"district_id\" = \"T2\".\"district_id\" WHERE \"T2\".\"A3\" = 'south Moravia' AND \"T1\".\"frequency\" = 'POPLATEK MESICNE'" + }, + { + "id": "eval_004", + "database": "financial", + "nlq": "How many accounts who have region in south Bohemia are eligible for loans?", + "golden_sql": "SELECT COUNT(\"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"loan\" AS \"T2\" ON \"T1\".\"account_id\" = \"T2\".\"account_id\" INNER JOIN \"district\" AS \"T3\" ON \"T1\".\"district_id\" = \"T3\".\"district_id\" WHERE \"T3\".\"A3\" = 'south Bohemia'" + } +] diff --git a/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml new file mode 100644 index 00000000..c66f2491 --- /dev/null +++ b/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: astana-transformation +region: us-east4 +cluster: juexinw-test +instance: juexinw-test-primary +database: financial +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/core-cujs/workspace_post_bootstrap/autoctx/state.md b/evals/core-cujs/workspace_post_bootstrap/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/core-cujs/workspace_post_bootstrap/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/bootstrap_context.json b/evals/core-cujs/workspace_post_bootstrap/bootstrap_context.json similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/bootstrap_context.json rename to evals/core-cujs/workspace_post_bootstrap/bootstrap_context.json diff --git a/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml new file mode 100644 index 00000000..c66f2491 --- /dev/null +++ b/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: astana-transformation +region: us-east4 +cluster: juexinw-test +instance: juexinw-test-primary +database: financial +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/core-cujs/workspace_post_dataset_generation/autoctx/state.md b/evals/core-cujs/workspace_post_dataset_generation/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/core-cujs/workspace_post_dataset_generation/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md new file mode 100644 index 00000000..3d1b3bcf --- /dev/null +++ b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md @@ -0,0 +1,16 @@ +# Hill-Climbing Experiment: my-alloydb-tuning-experiment + +## Metadata +- Workspace: ./.context-engineering/experiments/my-alloydb-tuning-experiment/ +- Golden dataset: ./golden.json +- DB source: my-alloydb +- Context Store: projects/cloud-db-nl2sql/locations/us-east1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx + +## Iteration Log + +### v0 (base) +- Context resource: projects/cloud-db-nl2sql/locations/us-east1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx@v0 +- Score: 0.75 +- Analysis: (baseline — no analysis) +- Eval report: v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/ +- Notes: baseline uploaded from bootstrap_context.json diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/context_set_v0.json b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/context_set_v0.json new file mode 100644 index 00000000..2b831e83 --- /dev/null +++ b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/context_set_v0.json @@ -0,0 +1,44 @@ +{ + "templates": [ + { + "nl_query": "How many accounts are there?", + "sql": "SELECT count(*) FROM public.account", + "intent": "Count total accounts", + "manifest": "Count total accounts", + "parameterized": { + "parameterized_sql": "SELECT count(*) FROM public.account", + "parameterized_intent": "Count total accounts" + } + }, + { + "nl_query": "What is the average loan amount?", + "sql": "SELECT avg(amount) FROM public.loan", + "intent": "Calculate average loan amount", + "manifest": "Calculate average loan amount", + "parameterized": { + "parameterized_sql": "SELECT avg(amount) FROM public.loan", + "parameterized_intent": "Calculate average loan amount" + } + } + ], + "facets": [ + { + "sql_snippet": "public.loan.amount > 100000", + "intent": "high amount loans (above 100,000)", + "manifest": "high amount loans (above a given number)", + "parameterized": { + "parameterized_sql_snippet": "public.loan.amount > 100000", + "parameterized_intent": "high amount loans (above 100,000)" + } + }, + { + "sql_snippet": "public.client.gender = 'F'", + "intent": "female clients", + "manifest": "female clients", + "parameterized": { + "parameterized_sql_snippet": "public.client.gender = 'F'", + "parameterized_intent": "female clients" + } + } + ] +} diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/db_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/db_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/db_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/db_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/golden_queries.json b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/golden_queries.json similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/golden_queries.json rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/golden_queries.json diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/llmrater_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/llmrater_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/llmrater_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/llmrater_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/model_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/model_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/model_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/model_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/run_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/run_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/run_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/run_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv diff --git a/evals/core-cujs/workspace_post_bootstrap/autoctx/tools.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/tools.yaml similarity index 100% rename from evals/core-cujs/workspace_post_bootstrap/autoctx/tools.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/tools.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/state.md b/evals/core-cujs/workspace_post_evaluation/autoctx/state.md deleted file mode 100644 index c164c1bc..00000000 --- a/evals/core-cujs/workspace_post_evaluation/autoctx/state.md +++ /dev/null @@ -1,9 +0,0 @@ -# Autoctx State - -Active Experiment: my-alloydb-tuning-experiment - -## History -- **Experiment:** my-alloydb-tuning-experiment - - Status: Evaluating - - Context Set ID: projects/cloud-db-nl2sql/locations/us-east1/contextSets/whaoyu-eval-contextset - - Golden Dataset: golden.json diff --git a/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml new file mode 100644 index 00000000..c66f2491 --- /dev/null +++ b/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: astana-transformation +region: us-east4 +cluster: juexinw-test +instance: juexinw-test-primary +database: financial +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/core-cujs/workspace_post_init/autoctx/experiments/.gitkeep b/evals/core-cujs/workspace_post_init/.context-engineering/experiments/.gitkeep similarity index 100% rename from evals/core-cujs/workspace_post_init/autoctx/experiments/.gitkeep rename to evals/core-cujs/workspace_post_init/.context-engineering/experiments/.gitkeep diff --git a/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml new file mode 100644 index 00000000..c66f2491 --- /dev/null +++ b/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: astana-transformation +region: us-east4 +cluster: juexinw-test +instance: juexinw-test-primary +database: financial +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/core-cujs/workspace_post_init/autoctx/state.md b/evals/core-cujs/workspace_post_init/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/core-cujs/workspace_post_init/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/smoke-test/workspace_post_init/autoctx/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_grounding_github_code/.context-engineering/experiments/.gitkeep similarity index 100% rename from evals/smoke-test/workspace_post_init/autoctx/experiments/.gitkeep rename to evals/eval-dataset-gen/workspace_grounding_github_code/.context-engineering/experiments/.gitkeep diff --git a/evals/freeform-input/workspace_missing_info_exploration/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_grounding_github_code/.context-engineering/tools.yaml similarity index 97% rename from evals/freeform-input/workspace_missing_info_exploration/autoctx/tools.yaml rename to evals/eval-dataset-gen/workspace_grounding_github_code/.context-engineering/tools.yaml index 2624bc1e..cac5a034 100644 --- a/evals/freeform-input/workspace_missing_info_exploration/autoctx/tools.yaml +++ b/evals/eval-dataset-gen/workspace_grounding_github_code/.context-engineering/tools.yaml @@ -5,7 +5,7 @@ project: cloud-db-nl2sql region: us-central1 cluster: whaoyu-test instance: whaoyu-test-primary -database: financial +database: search --- kind: tool name: my-alloydb-list-schemas diff --git a/evals/eval-dataset-gen/workspace_grounding_local_code/.context-engineering/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_grounding_local_code/.context-engineering/experiments/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/evals/smoke-test/workspace_post_init/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_grounding_local_code/.context-engineering/tools.yaml similarity index 97% rename from evals/smoke-test/workspace_post_init/autoctx/tools.yaml rename to evals/eval-dataset-gen/workspace_grounding_local_code/.context-engineering/tools.yaml index 2624bc1e..b01c55fd 100644 --- a/evals/smoke-test/workspace_post_init/autoctx/tools.yaml +++ b/evals/eval-dataset-gen/workspace_grounding_local_code/.context-engineering/tools.yaml @@ -5,7 +5,7 @@ project: cloud-db-nl2sql region: us-central1 cluster: whaoyu-test instance: whaoyu-test-primary -database: financial +database: db_hr --- kind: tool name: my-alloydb-list-schemas diff --git a/evals/eval-dataset-gen/workspace_grounding_querylog/.context-engineering/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_grounding_querylog/.context-engineering/experiments/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/evals/core-cujs/workspace_post_dataset_generation/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_grounding_querylog/.context-engineering/tools.yaml similarity index 100% rename from evals/core-cujs/workspace_post_dataset_generation/autoctx/tools.yaml rename to evals/eval-dataset-gen/workspace_grounding_querylog/.context-engineering/tools.yaml diff --git a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/experiments/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/tools.yaml b/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/tools.yaml new file mode 100644 index 00000000..30cbe22e --- /dev/null +++ b/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary +database: petstore +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/experiments/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/tools.yaml b/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/tools.yaml new file mode 100644 index 00000000..6cc51853 --- /dev/null +++ b/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/.context-engineering/tools.yaml @@ -0,0 +1,33 @@ +kind: source +name: my-alloydb +type: alloydb-postgres +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary +database: db_ecommerce_cryptic +--- +kind: tool +name: my-alloydb-list-schemas +type: postgres-list-tables +source: my-alloydb +description: | + Use this tool to list tables and their schemas in the my-alloydb database. + + Progressive Schema Discovery (Recommended): + 1) Fetch structure first (output_format='simple'), + 2) Go deep on specific parts if interested, + 3) Use batching if info is too large. + + Scope: + - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. + + Behavior: + - Omit 'table_names' to fetch all tables. + - Omit 'output_format' for detailed schema (default). +--- +kind: tool +name: my-alloydb-execute-sql +type: postgres-execute-sql +source: my-alloydb +description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/freeform-input/dataset.json b/evals/freeform-input/dataset.json index 2db36fb6..23673c5e 100644 --- a/evals/freeform-input/dataset.json +++ b/evals/freeform-input/dataset.json @@ -3,7 +3,7 @@ { "id": "freeform:explain:capabilities", "starting_prompt": "What kind of context can you generate for me?", - "conversation_plan": "Verify that the agent explains Templates, Facets, and Value Searches based on the `context-generation-guide` skill. Terminate after the agent provides the explanation.", + "conversation_plan": "Verify that the agent explains Templates, Facets, and Value Searches based on the `context-engineering-generation-guide` skill. Terminate after the agent provides the explanation.", "kind": "agents", "work_dir": "workspace_explain_capabilities/", "max_turns": 5 @@ -35,7 +35,7 @@ { "id": "freeform:targeted:template", "starting_prompt": "Create a template for: 'How many accounts are in region Prague?' with SQL: 'SELECT count(*) FROM account AS T1 INNER JOIN district AS T2 ON T1.district_id = T2.district_id WHERE T2.A3 = ''Prague'''", - "conversation_plan": "Verify that the agent uses the `context-generation-guide` skill. Check that it correctly parameterizes the query (e.g., using `$1` for 'Prague') and returns a valid Template JSON object with nl_query, sql, intent, manifest, and parameterized fields. Terminate after the agent provides the JSON.", + "conversation_plan": "Verify that the agent uses the `context-engineering-generation-guide` skill. Check that it correctly parameterizes the query (e.g., using `$1` for 'Prague') and returns a valid Template JSON object with nl_query, sql, intent, manifest, and parameterized fields. Terminate after the agent provides the JSON.", "kind": "agents", "work_dir": "workspace_targeted_template/", "max_turns": 5 @@ -43,7 +43,7 @@ { "id": "freeform:targeted:facet", "starting_prompt": "Create a facet for: 'issuance after transaction' with SQL snippet: `account.frequency = 'POPLATEK PO OBRATU'`", - "conversation_plan": "Verify that the agent uses the `context-generation-guide` skill. Check that it returns a valid Facet JSON object with sql_snippet: \"account.frequency = 'POPLATEK PO OBRATU'\" (column reference must be table-qualified as `account.frequency`, not bare `frequency`), intent: \"issuance after transaction\", manifest: \"issuance after transaction frequency\", and appropriate parameterized fields (e.g., `account.frequency = $1`). Terminate after the agent provides the JSON.", + "conversation_plan": "Verify that the agent uses the `context-engineering-generation-guide` skill. Check that it returns a valid Facet JSON object with sql_snippet: \"account.frequency = 'POPLATEK PO OBRATU'\" (column reference must be table-qualified as `account.frequency`, not bare `frequency`), intent: \"issuance after transaction\", manifest: \"issuance after transaction frequency\", and appropriate parameterized fields (e.g., `account.frequency = $1`). Terminate after the agent provides the JSON.", "kind": "agents", "work_dir": "workspace_targeted_facet/", "max_turns": 5 @@ -51,7 +51,7 @@ { "id": "freeform:targeted:value-search", "starting_prompt": "Create a value search for 'Region' in `district.A3` table using Trigram match.", - "conversation_plan": "Verify that the agent uses the `context-generation-guide` skill. Check that it returns a valid ValueSearch JSON object. Terminate after the agent provides the JSON.", + "conversation_plan": "Verify that the agent uses the `context-engineering-generation-guide` skill. Check that it returns a valid ValueSearch JSON object. Terminate after the agent provides the JSON.", "kind": "agents", "work_dir": "workspace_targeted_value_search/", "max_turns": 5 diff --git a/evals/core-cujs/workspace_post_evaluation/autoctx/tools.yaml b/evals/freeform-input/workspace_fix_query_gap/.context-engineering/tools.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/autoctx/tools.yaml rename to evals/freeform-input/workspace_fix_query_gap/.context-engineering/tools.yaml diff --git a/evals/core-cujs/workspace_post_init/autoctx/tools.yaml b/evals/freeform-input/workspace_missing_info_exploration/.context-engineering/tools.yaml similarity index 100% rename from evals/core-cujs/workspace_post_init/autoctx/tools.yaml rename to evals/freeform-input/workspace_missing_info_exploration/.context-engineering/tools.yaml diff --git a/evals/smoke-test/dataset.json b/evals/smoke-test/dataset.json index 3efb5bf3..d506ad0f 100644 --- a/evals/smoke-test/dataset.json +++ b/evals/smoke-test/dataset.json @@ -3,7 +3,7 @@ { "id": "plugin:list-skills", "starting_prompt": "Show the FULL list of Agent Skills you have available", - "conversation_plan": "Verify the db-context-engineering plugin loaded correctly. The agent should list its db-context-engineering skills (e.g., context-engineering-workflow, context-generation-guide). Terminate the conversation immediately after the agent provides the list.", + "conversation_plan": "Verify the db-context-engineering plugin loaded correctly. The agent should list its db-context-engineering skills (e.g., context-engineering-workflow, context-engineering-generation-guide). Terminate the conversation immediately after the agent provides the list.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_init/", @@ -21,7 +21,7 @@ { "id": "plugin:list-skills-and-mcp-tools", "starting_prompt": "Show the FULL list of Agent Skills and MCP tools you have available", - "conversation_plan": "Verify the db-context-engineering plugin loaded correctly. The agent should list its db-context-engineering skills (e.g., context-engineering-workflow, context-generation-guide) and the MCP tools provided by the two plugin-configured servers: the `db-context-engineering` MCP server (e.g., generate_dataset, generate_evalbench_configs, generate_upload_url, mutate_context_set, read_evaluation_result) and the `toolbox` MCP server (*-list-schemas, *-execute-sql). Terminate the conversation immediately after the agent provides the list.", + "conversation_plan": "Verify the db-context-engineering plugin loaded correctly. The agent should list its db-context-engineering skills (e.g., context-engineering-workflow, context-engineering-generation-guide) and the MCP tools provided by the two plugin-configured servers: the `db-context-engineering` MCP server (e.g., generate_dataset, generate_evalbench_configs, generate_upload_url, mutate_context_set, read_evaluation_result) and the `toolbox` MCP server (*-list-schemas, *-execute-sql). Terminate the conversation immediately after the agent provides the list.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_init/", diff --git a/evals/smoke-test/workspace_post_init/.context-engineering/experiments/.gitkeep b/evals/smoke-test/workspace_post_init/.context-engineering/experiments/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/evals/freeform-input/workspace_fix_query_gap/autoctx/tools.yaml b/evals/smoke-test/workspace_post_init/.context-engineering/tools.yaml similarity index 100% rename from evals/freeform-input/workspace_fix_query_gap/autoctx/tools.yaml rename to evals/smoke-test/workspace_post_init/.context-engineering/tools.yaml diff --git a/evals/smoke-test/workspace_post_init/autoctx/state.md b/evals/smoke-test/workspace_post_init/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/smoke-test/workspace_post_init/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/gemini-extension.json b/gemini-extension.json index 1eded213..c39cc5b1 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -14,7 +14,7 @@ "args": [ "toolbox-server@1.4.0", "--config", - "autoctx/tools.yaml", + ".context-engineering/tools.yaml", "--stdio" ] } diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json index 54c100c8..e9247101 100644 --- a/plugin/.claude-plugin/plugin.json +++ b/plugin/.claude-plugin/plugin.json @@ -17,7 +17,7 @@ "args": [ "toolbox-server@1.4.0", "--config", - "autoctx/tools.yaml", + ".context-engineering/tools.yaml", "--stdio" ] } diff --git a/plugin/skills/context-engineering-bootstrap/SKILL.md b/plugin/skills/context-engineering-bootstrap/SKILL.md index c90a1ee2..81e5e959 100644 --- a/plugin/skills/context-engineering-bootstrap/SKILL.md +++ b/plugin/skills/context-engineering-bootstrap/SKILL.md @@ -1,70 +1,53 @@ --- name: context-engineering-bootstrap -description: Guides the agent to bootstrap an initial ContextSet (templates, facets, and value searches) by deducing key information from the database schema and generating a ContextSet file. +description: Generate a baseline ContextSet (Templates, Facets, Value Searches) from a target database's schema (and optional design docs / application code) and save to a caller-specified path. Optionally upload to the Context Store. --- -> **Load the `context-engineering-workflow` skill first.** It holds the shared context this phase depends on: workspace layout, state file conventions, phase order, and safety protocol. Do not proceed with this phase without reading it. +> **Load [`context-engineering-workflow`](../context-engineering-workflow/SKILL.md) first** for shared terminology, lifecycle overview, and safety protocol. -> [!NOTE] -> For detailed schema specifications and explanation of context set types, see the central [Context Set Concept Types](../context-generation-guide/SKILL.md) guide. - -# Phase: Baseline Bootstrapping +# Skill: Baseline ContextSet Bootstrapping ## Goal -Deduce query concepts and generate a baseline `ContextSet` (templates, facets, value searches) directly from database schemas and metadata to act as the starting point for optimization. - -## Input - -Before beginning the workflow, you explicitly require: -- An active `tools.yaml` configuration (located in `autoctx/`) with database schema fetching tools configured (e.g., `-list-schemas`). -- Target database schemas to act upon. - -## Workflow - -Follow these steps exactly in order: - -1. **Condition Check & Schema Retrieval:** - - **Ask for Experiment Name & Handle Existing Folders**: You must explicitly ask the user for a descriptive name for this tuning experiment (e.g., `sales_db_tuning`). - - **If the experiment folder already exists inside `autoctx/experiments/`**: You **MUST** detect it and explicitly ask the user for confirmation: - - *"An experiment named `` already exists. Do you want to resume it (update its baseline context), fork it (create a new version, e.g., `_v2`), or overwrite it completely?"* - - If the user selects **resume**: proceed with the bootstrap in the same folder, updating `bootstrap_context.json`. - - If the user selects **fork**: prompt for a new name or suggest `_v2`, create the folder, and proceed there. - - If the user selects **overwrite**: clear the existing folder's contents and proceed. - - **If it does not exist**: Create a new dedicated subfolder inside `autoctx/experiments/` using this name. - - Do not proceed until the experiment folder structure is finalized. - - Use the available Toolbox MCP tools configured in the active `autoctx/tools.yaml` to fetch the schemas for the target database. - - Present the retrieved schema summary **structurally and cleanly** to the user. Ask the user if they want to filter or focus on specific schemas or tables. - - **Source Enrichment**: Prompt the user for any existing **Design Docs** or **Application Code** (e.g., ORM models, SQL queries) they wish to provide to enrich the context generation. Wait for the user's response before proceeding. - -2. **Deduce Key Info (Core Execution):** - - Perform a **deep analysis** of the retrieved **schema and any provided documentation or code** to identify important concepts, relationships, and likely query patterns. - - **Collect Candidates**: Identify representative natural language queries with their corresponding SQL, common filter conditions or business rules, and **columns that require specialized value searching** (e.g., names needing fuzzy match, descriptions needing semantic search). - - *Review Check:* Briefly display these candidates to the user for approval or modifications before proceeding. - -3. **Context Generation (Core Execution):** - - **Invoke the `context-generation-guide` skill** to produce the context (Templates, Facets, and Value Searches). - - Provide the deduced candidates collected in Step 2 as input to that skill. - - That skill will handle phrase extraction, parameterization, and constructing the final valid JSON structure according to dialect best practices for all context types. - - Once generated, use the `mutate_context_set` MCP tool to save the context items to `bootstrap_context.json` inside the approved experiment folder. Since this is a new file, construct a list of `"operation": "add"` mutations for each generated item (Template, Facet, Value Search) and pass them to the tool. - -## Output - -Upon successful completion, the workspace must contain: -- A generated `.json` file (`bootstrap_context.json`) representing the baseline `ContextSet`, stored successfully at the requested `output_file_path`. - -## Upload Advice & Next Steps - -Conclude by providing a succinct summary to the user: -1. **Summarize Results**: - - Confirm that the bootstrap context file has been successfully generated and saved. - - Mention the final file path. -2. **Upload Instructions**: - - **Read Database Details**: Read `autoctx/tools.yaml` to fetch the specific project, location, and instance/cluster details for the active database. - - **Generate URL**: Call the `generate_upload_url` tool passing the extracted values to provide the direct console link to the user. - - Present the local file path to `bootstrap_context.json` and the generated console link together in a single clear message. -3. **Instruct Next Step Evaluation**: - - Instruct the user to upload the file to Database Studio and then run evaluation using the evaluating workflow on this new ContextSet to establish a baseline. - - -> [!IMPORTANT] -> **Tool Modification Rule**: Always use the `mutate_context_set` tool for all ContextSet changes. Pass mutation payloads directly to the tool — it handles all file I/O internally. **Do not read the target context set file beforehand**. +From a target database and optional user-supplied enrichment sources (design docs, ORM models, sample SQL, glossary), produce a baseline `ContextSet` JSON at a caller-specified path. Optionally upload to the Context Store and return the resource name. + +## Prerequisites +- A working DB connection — Toolbox MCP tools (`-list-schemas`) must be visible to the agent throughout the run. If missing or unreachable at any point, stop and route through `context-engineering-init`; do not work around it (no bash `uvx toolbox-server invoke` fallback). +- Target output path for the ContextSet JSON. If not supplied, prompt the user; default `./bootstrap_context.json` at cwd. +- (Optional) Design docs, application code, sample SQL, glossary, or other enrichment sources. +- (Optional, for upload) Context Store resource coordinates — see the `upload_context_set` tool for required fields. + +## Guidance + +1. **Confirm scope with the user:** + - Which Toolbox `` to introspect (auto-select if exactly one supported source exists in `tools.yaml`; otherwise prompt). + - Which schemas / tables to focus on (or all, if the DB is small). + - Output path for the ContextSet JSON. + - Whether to upload to Context Store after generation; if yes, collect the resource coordinates required by `upload_context_set`. + +2. **Fetch the schema:** use `-list-schemas` (and related introspection tools). Present the schema summary structurally to the user. + +3. **Collect enrichment sources:** prompt for design docs, ORM models, sample SQL, glossary, etc. Wait for the user's response before proceeding. + +4. **Identify candidate items:** analyze schema + enrichment to identify representative NLQ + SQL pairs (Templates), common filter fragments (Facets), and columns needing fuzzy/semantic matching (Value Searches). Present the candidates to the user for review before generating. + +5. **Generate the ContextSet:** invoke the `context-engineering-generation-guide` skill with the approved candidates. Save items incrementally to the output path via the `mutate_context_set` MCP tool. For a new file, construct `"operation": "add"` mutations for each item. + +6. **Optionally upload:** if the user opted to upload, call `upload_context_set`. + +7. **Summarize:** report the local file path and (if uploaded) the resource name. + +## Rules +- Caller supplies (or explicitly confirms a default for) the output path. +- Never upload without explicit user consent. +- Always use the `mutate_context_set` MCP tool for ContextSet file changes — pass mutation payloads directly. Do not read the target file beforehand. +- Do not invoke `context-engineering-evaluate` or `context-engineering-hillclimb`. + +## Tools + +**MCP:** +- `-list-schemas` (Toolbox) — schema introspection. +- `mutate_context_set` — incremental writes to the output JSON. +- `upload_context_set` → `cs_resource_name` — optional Context Store upload. + +**Sibling skill:** +- `context-engineering-generation-guide` — produces well-formed Template / Facet / Value Search JSON. Also the reference for context-item schema and authoring standards. diff --git a/plugin/skills/context-engineering-evaluate/SKILL.md b/plugin/skills/context-engineering-evaluate/SKILL.md index 5cb0668b..edec30b0 100644 --- a/plugin/skills/context-engineering-evaluate/SKILL.md +++ b/plugin/skills/context-engineering-evaluate/SKILL.md @@ -3,25 +3,22 @@ name: context-engineering-evaluate description: Guides the agent to execute an evaluation of a ContextSet against a golden NLQ+SQL dataset using the Evalbench framework. --- -> **Load the `context-engineering-workflow` skill first.** It holds the shared context this phase depends on: workspace layout, state file conventions, phase order, and safety protocol. Do not proceed with this phase without reading it. +> **Load [`context-engineering-workflow`](../context-engineering-workflow/SKILL.md) first** for shared terminology, lifecycle overview, and safety protocol. -# Phase: Evaluation Scoring +# Skill: Evaluation Scoring ## Goal -Run a structured Evalbench evaluation to score the accuracy of a specific context set against a golden truth dataset, identifying exact query failures. +Score a ContextSet against a golden dataset by running Evalbench, and return a scored report — overall accuracy plus dominant failure categories. -## Input +## Prerequisites -Before beginning the workflow, you explicitly require: -- A `tools.yaml` file securely located in the workspace root directory containing the target database connection details. -- A golden evaluation dataset (`golden_dataset_path`), formatted as an absolute system path. The file must be in the **simplified user-facing format**. - - **Simplified User-Facing Dataset Format**: - A JSON list of objects, where each object must have the following keys: - - `id`: Unique string identifier (e.g., `eval_001`). - - `database`: Target database name. - - `nlq`: Natural language question. - - `golden_sql`: The correct reference SQL query. +- A working DB connection (`tools.yaml` configured for the Toolbox MCP server — see `context-engineering-init` if missing). +- ADC configured and the Gemini Data Analytics + Dataplex APIs enabled on the project (see `context-engineering-init` for preflight). +- A golden evaluation dataset (absolute path) in the **simplified user-facing format** — a JSON list of objects, each with: + - `id`: unique identifier (e.g., `eval_001`). + - `database`: target database name. + - `nlq`: natural language question. + - `golden_sql`: correct reference SQL query. Example: ```json @@ -34,61 +31,56 @@ Before beginning the workflow, you explicitly require: } ] ``` -- The `context_set_id` (the Data Agent's authored context configuration identifier, retrievable by the user directly from the GCP Database Studio console; e.g., `projects//locations//contextSets/`). - -## Workflow - -Follow these steps exactly in order: - -1. **Experiment Selection & Memory:** - - Scan the local `autoctx/experiments/` directory and list the available tuning workflows/subfolders to the user. - - **If no experiment folders exist** (or the user wants to create a new one without running Bootstrap): - - **Ask the user to choose** between 2 paths (do not assume): - 1. **Bootstrap a basic context**: Guide them to trigger the Bootstrap workflow. - 2. **Use an existing context**: - - > [!IMPORTANT] - > Inform the user that if they have an existing context, it must be uploaded to GCP Database Studio to obtain a `context_set_id` for evaluation. - - Ask the user for a name for this new experiment folder (similar to how Bootstrap does). - - Create the folder under `autoctx/experiments/`. - - Ask the user to provide the local file path of their existing context. - - Record the local file path as the Base Context for this experiment in `autoctx/state.md` for long-term memory. - - Continue with the evaluation flow below. - - Wait for the user to explicitly select an experiment folder to evaluate (or use the newly created one). - - Once selected, explicitly record their chosen experiment name into the local `autoctx/state.md` file to act as long-term memory so you don't forget it during subsequent evaluations. - -2. **Parameter Collection:** - - **User Inputs:** Prompt the user ONLY for the `golden_dataset_path` and the `context_set_id` (if they haven't provided them already). Do NOT ask them to explain or verify database configurations. - - **Interactive DB Selection:** Read the `autoctx/tools.yaml` file to list available databases to the user: - 1. Find all `kind: source` blocks with supported evaluation engines (consult the `generate_evalbench_configs` tool description for the exact list of supported types). - 2. If there is exactly one *supported* source, inform the user and auto-select it. - 3. If there are multiple *supported* sources, list their `name` and `type` and let the user select which database to evaluate. - -3. **Config Generation (Core Execution):** - - Use the `generate_evalbench_configs` MCP tool. This is the **only** way to generate Evalbench configs. Never invent configs from scratch. - - If the tool fails, analyze the error and retry with corrected inputs. If it is an internal system error, STOP and inform the user. - - Provide the selected `output_dir` (must be `autoctx/experiments//`), `dataset_path`, `context_set_id`, absolute `toolbox_config_path` (e.g. `autoctx/tools.yaml`), and selected `toolbox_source_name`. - - The tool will automatically write all generated configuration files (including `golden_queries.json`) directly to the `eval_configs/` directory inside the chosen `autoctx/experiments//` folder. - - You do not need to manually write or extract file contents. Verify that the files have materialized if needed. - -4. **Evalbench Run Integration:** - - Trigger the `run_shell_command` natively to execute the evaluation from the ROOT of the workspace using the following exact command template: - `uvx google-evalbench@1.9.0 --experiment_config=autoctx/experiments//eval_configs/run_config.yaml` - - Check the command outputs to ensure the evaluation reports materialize in the respective `autoctx/experiments//eval_reports/` directory. - -## Output - -Upon successful completion, the workspace must contain: -- The generated Evalbench config files successfully written to the `eval_configs/` folder. -- Evaluating reports built successfully by the external Evalbench runner process. - -## Final Summary & Next Steps - -Conclude by providing a succinct summary to the user: -- Confirm that the context set has been scored and point out exactly where the final metrics CSV/results are located. -- Share top-level performance summaries. -- Suggest actionable next steps (e.g., transition to a refinement workflow to hill-climb and improve the metrics based on failed evaluations). - -## Templates & Reference - -When listing sources from `tools.yaml`, ensure you only present `kind: source` records to the user. -The tool `generate_evalbench_configs` will find the selected block inside the file and validate its connection parameters deterministically using Python code. You do not need to manually parse or map individual properties such as `host`, `port`, or `database` yourself. If the tool indicates a verification failure for a specific database type, refer to the schema examples inside this directory (e.g., `cloud-sql-postgres.md`) to guide the user on fixing their `tools.yaml` definition. + +- A ContextSet, supplied as **exactly one of**: + - **`cs_resource_name`** — a full Context Store resource name (e.g., `projects/

/locations//contextSetGroups//contextSets/@`). Used directly. + - **Local ContextSet JSON file** + the coordinates required by `upload_context_set` — the skill uploads it (with explicit user consent) to obtain a `cs_resource_name` for the run. + +- An `output_dir` (absolute or workspace-relative) where the eval configs and reports should live. If the user hasn't specified one, prompt them; a sensible suggestion is `./eval-runs//`. + +## Guidance + +1. **Collect inputs.** Prompt only for what's missing from the Prerequisites. Trust `tools.yaml` values as-is — don't ask the user to re-verify them. + +2. **Prepare the ContextSet resource name.** + - If the user supplied a `cs_resource_name`, use it directly. + - If the user supplied a local file: + - Confirm every field required by `upload_context_set` is present. Ask for any missing values individually — do not guess. + - Ask for explicit consent before uploading. Summarize the target resource in the prompt so the user knows what will be written. + - On consent, call `upload_context_set`. The returned resource name becomes `cs_resource_name` for the rest of this run. + - On `upload_context_set` failure, surface the error verbatim and stop. Do not fall back to a manual upload URL. + +3. **Select the DB source from `tools.yaml`.** + - Find all `kind: source` blocks whose `type` is a supported evaluation engine (consult `generate_evalbench_configs` for the current list). + - If exactly one supported source exists, inform the user and auto-select it. + - If multiple, list their `name` + `type` and let the user pick. + +4. **Generate the Evalbench configs.** Call `generate_evalbench_configs`. The tool writes configs under `/eval_configs/`. This is the only supported way to produce Evalbench configs — never author them by hand. + +5. **Run Evalbench.** Shell out from the caller's cwd: + `uvx google-evalbench@1.10.0 --experiment_config=/eval_configs/run_config.yaml` + Reports materialize under `/eval_reports//` (the job_id appears in the tool's stdout). The run can take many minutes for larger datasets — let it complete; do not kill or restart on apparent stalls. Treat a non-zero exit code as a hard failure and surface stderr verbatim. + +6. **Read and summarize results.** Call `read_evaluation_result` on the run folder `/eval_reports//`. Report to the user: overall score, dominant failure categories, and the absolute path to the full reports. Suggest hillclimb as a natural next step if the user wants to iteratively improve. + +## Rules + +- Never upload a ContextSet without explicit user consent. +- Never invoke bootstrap or hillclimb from within this skill. +- If both `cs_resource_name` and a local file are provided, ask the user which to use — do not silently pick. +- On `generate_evalbench_configs` errors, surface the error and stop; do not retry blindly. +- This skill is stateless. Every path comes from the caller — do not assume a workspace layout or write cross-phase state files. +- Use the caller's Context Store coordinates (`project_id`, `csg_id`, `cs_id`, `version`) verbatim when supplied. If any are missing, ask the user explicitly — do not infer from filenames, paths, or `tools.yaml` without their confirmation. + +## Tools + +**MCP:** +- `upload_context_set` → `cs_resource_name` — used only when the caller supplies a local file instead of a resource name. +- `generate_evalbench_configs` — produces Evalbench YAML configs on disk. +- `read_evaluation_result` — parses `scores.csv` / `summary.csv` into a markdown summary. + +**Shell:** +- `uvx google-evalbench@1.10.0 --experiment_config=` — runs the eval job against the published release. + +**References:** +- `references/.md` (`alloydb-postgres.md`, `cloud-sql-mysql.md`, `cloud-sql-postgres.md`, `spanner.md`) — per-engine schema examples for fixing `tools.yaml` source blocks if `generate_evalbench_configs` reports a validation failure. diff --git a/plugin/skills/context-generation-guide/SKILL.md b/plugin/skills/context-engineering-generation-guide/SKILL.md similarity index 99% rename from plugin/skills/context-generation-guide/SKILL.md rename to plugin/skills/context-engineering-generation-guide/SKILL.md index 2b30f479..db4a50fc 100644 --- a/plugin/skills/context-generation-guide/SKILL.md +++ b/plugin/skills/context-engineering-generation-guide/SKILL.md @@ -1,10 +1,9 @@ --- -name: context-generation-guide +name: context-engineering-generation-guide description: Guidelines and best practices for generating context items (Templates, Facets, Value Searches). Use this skill whenever the user asks to create, author, or generate context for database enrichment, or asks for examples and instructions on how to write templates, facets, or value searches. It helps bridge the gap between LLMs and structured databases. For running the automated generation, evaluation, and tuning lifecycle, see the context-engineering-workflow skill. --- -# Context Generation Guide Skill - +# Skill: Context Generation Guide This skill provides the agent with the necessary information, concepts, and best practices to generate high-quality context items for the "Context Engineering Agent". This context bridges the gap between LLMs and structured databases, enabling accurate Natural Language to SQL generation. diff --git a/plugin/skills/context-generation-guide/references/facet/googlesql.md b/plugin/skills/context-engineering-generation-guide/references/facet/googlesql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/facet/googlesql.md rename to plugin/skills/context-engineering-generation-guide/references/facet/googlesql.md diff --git a/plugin/skills/context-generation-guide/references/facet/mysql.md b/plugin/skills/context-engineering-generation-guide/references/facet/mysql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/facet/mysql.md rename to plugin/skills/context-engineering-generation-guide/references/facet/mysql.md diff --git a/plugin/skills/context-generation-guide/references/facet/postgresql.md b/plugin/skills/context-engineering-generation-guide/references/facet/postgresql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/facet/postgresql.md rename to plugin/skills/context-engineering-generation-guide/references/facet/postgresql.md diff --git a/plugin/skills/context-generation-guide/references/phrase_extraction/guidelines.md b/plugin/skills/context-engineering-generation-guide/references/phrase_extraction/guidelines.md similarity index 100% rename from plugin/skills/context-generation-guide/references/phrase_extraction/guidelines.md rename to plugin/skills/context-engineering-generation-guide/references/phrase_extraction/guidelines.md diff --git a/plugin/skills/context-generation-guide/references/template/googlesql.md b/plugin/skills/context-engineering-generation-guide/references/template/googlesql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/template/googlesql.md rename to plugin/skills/context-engineering-generation-guide/references/template/googlesql.md diff --git a/plugin/skills/context-generation-guide/references/template/mysql.md b/plugin/skills/context-engineering-generation-guide/references/template/mysql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/template/mysql.md rename to plugin/skills/context-engineering-generation-guide/references/template/mysql.md diff --git a/plugin/skills/context-generation-guide/references/template/postgresql.md b/plugin/skills/context-engineering-generation-guide/references/template/postgresql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/template/postgresql.md rename to plugin/skills/context-engineering-generation-guide/references/template/postgresql.md diff --git a/plugin/skills/context-generation-guide/references/value_search/googlesql.md b/plugin/skills/context-engineering-generation-guide/references/value_search/googlesql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/value_search/googlesql.md rename to plugin/skills/context-engineering-generation-guide/references/value_search/googlesql.md diff --git a/plugin/skills/context-generation-guide/references/value_search/mysql.md b/plugin/skills/context-engineering-generation-guide/references/value_search/mysql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/value_search/mysql.md rename to plugin/skills/context-engineering-generation-guide/references/value_search/mysql.md diff --git a/plugin/skills/context-generation-guide/references/value_search/postgresql.md b/plugin/skills/context-engineering-generation-guide/references/value_search/postgresql.md similarity index 100% rename from plugin/skills/context-generation-guide/references/value_search/postgresql.md rename to plugin/skills/context-engineering-generation-guide/references/value_search/postgresql.md diff --git a/plugin/skills/context-engineering-hillclimb/SKILL.md b/plugin/skills/context-engineering-hillclimb/SKILL.md index bcbbed15..7cf5ff48 100644 --- a/plugin/skills/context-engineering-hillclimb/SKILL.md +++ b/plugin/skills/context-engineering-hillclimb/SKILL.md @@ -1,161 +1,76 @@ --- name: context-engineering-hillclimb -description: Guides the agent to perform hill-climbing iterations to improve a ContextSet based on Evalbench evaluation results. +description: Autonomously improve a ContextSet by iterating evaluate → analyze → mutate → re-upload until convergence, given a golden dataset and (optionally) a base context. The user supplies inputs once; no per-iteration approval needed. --- -> **Load the `context-engineering-workflow` skill first.** It holds the shared context this phase depends on: workspace layout, state file conventions, phase order, and safety protocol. Do not proceed with this phase without reading it. +> **Load [`context-engineering-workflow`](../context-engineering-workflow/SKILL.md) first** for shared terminology, lifecycle overview, and safety protocol. -# Phase: Optimization & Hill-Climbing +# Skill: Automated Hill-Climbing ## Goal -Analyze evaluation failures to perform a Gap Analysis and apply targeted context mutations (refinements) to iteratively improve translation accuracy. - -## Workflow - -Follow these steps exactly in order: - -### 1. Setup & Loop - -1. **Validation**: - - Check if `autoctx/experiments/` directory and `autoctx/state.md` exist. If missing, warn the user that the workspace might not be initialized (suggest running the Setup & Connection phase of this skill first). - - Once an experiment is selected, verify it contains an `eval_reports/` folder. If missing, suggest running the Evaluation workflow first. -2. **Identify Experiment**: - - Read the local `autoctx/state.md` to identify the active experiment. - - If not found, ask the user to select an experiment folder from `autoctx/experiments/`. -3. **Determine Loop Version**: - - Scan the `autoctx/experiments//hillclimb/` folder for files matching `improved_context_v*.json`. - - Determine the loop version `vN` by finding the maximum `N` and using `N+1`. If the folder is empty, start at `v1`. -4. **Locate Base Context**: - - For `v1`: - - Check `autoctx/state.md` to see if a specific base context path was recorded for this experiment (e.g., during the Evaluation setup for user-provided contexts). - - If not found in `state.md`, default to the baseline generated by Bootstrap in the experiment folder. - - If still not found, ask the user for the absolute path to their base context file and record it as the Base Context in autoctx/state.md. - - For `vN` (where N > 1), the base context is `improved_context_v(N-1).json`. - - Verify the base context file exists. If missing, STOP and ask the user for the correct path. - ---- - -### 2. Phase 1: Gap Analysis - -1. **Validation**: - - Determine the target evaluation run folder under `eval_reports/`. If multiple folders exist, find the most recent one by modified time. **Prefer the latest run by default**, but list other available runs as well (peeking into their `summary.csv` or `configs.csv` to show timestamps/metrics for visual context). Ask the user to confirm the selection. - - Verify that the selected `eval_reports//` contains expected files (e.g., `scores.csv`, `summary.csv`). If missing or empty, STOP and inform the user. -2. **Read Evaluation Results**: Use the `read_evaluation_result` MCP tool passing the path to `eval_reports//`. -3. **Generate Gap Analysis Report (Batched)**: - - The tool returns a summary and a batch of failure cases (default limit 10). - - Iterate through the failure cases by calling the tool with increasing `offset` (0, 10, 20, ...) until all failed queries are analyzed. - - **First Batch (offset=0)**: Initialize the report file with the `# Gap Analysis Report - vN` header and `## Summary` section, followed by the analysis of the first batch under `## Failed Queries Detail`. - - **Subsequent Batches**: Call the tool with the next offset, analyze the new failures, and **append** them to the `## Failed Queries Detail` section. - - Use the following structure for the report: - - ```markdown - # Gap Analysis Report - vN - - ## Summary - - **Total Queries**: 10 - - **Passed**: 7 - - **Failed**: 3 - - **Pass Rate**: 70% - - ## Failed Queries Detail - - ### Query 1: "How many users registered in 2023?" - - **Error Category**: `[FilterError]` - - **Expected SQL**: `SELECT count(*) FROM users WHERE year = 2023` - - **Actual SQL**: `SELECT count(*) FROM users` (Missing filter) - - **Root Cause**: The LLM did not know about the `year` column or how to filter by year for this entity. - - **Proposed Mutation**: Add a facet for "Users by Year". - - ### Query 2: "Show me top selling products" - - **Error Category**: `[OrderingError]` - - **Expected SQL**: `SELECT name FROM products ORDER BY sales DESC LIMIT 5` - - **Actual SQL**: `SELECT name FROM products LIMIT 5` - - **Root Cause**: Missing ordering instruction in context. - - **Proposed Mutation**: Update the template for "Product Sales" to include ordering. - - ### Query 3: "Get users older than 30" - - **Error Category**: `[GoldenDataError]` - - **Expected SQL**: `SELECT * FROM users WHERE age >> 30` (Syntax error `>>` in golden SQL) - - **Actual SQL**: `SELECT * FROM users WHERE age > 30` - - **Root Cause**: Invalid syntax in golden dataset. - - **Proposed Mutation**: None. Flag to user to fix the evaluation dataset. - ``` -4. **Save Report**: You **MUST** physically write the report file to `autoctx/experiments//hillclimb/gap_analysis_vN.md`. If you are processing in batches, ensure you append to this file until all failed queries are documented. Do not merely output it in chat; it must exist on the file system. -5. **Log in State Tracking**: - - Update `autoctx/state.md` to record the mapping for Loop `vN` (Base Context <-> Eval Report <-> Gap Analysis). -6. **Human-in-the-Loop Review**: - - Inform the user that the Gap Analysis report has been successfully written to disk. - - Ask the user if they want to review, make any corrections, or add manual feedback directly to the file before proceeding to Phase 2 (Context Mutation). - - Wait for user confirmation before starting Phase 2. - ---- - -### 3. Phase 2: Context Mutation - -Refer to [context-generation-guide](../context-generation-guide/SKILL.md) for how to edit a ContextSet. - -1. **Validation**: Verify that `gap_analysis_vN.md` exists and contains findings. Verify the base ContextSet file exists. If missing, STOP and inform the user. -2. **Analyze Gap Report & Determine Fixing Strategy**: - - Read `gap_analysis_vN.md` to identify what needs to be fixed. - - **Fixing Strategy Guidelines**: - - **Conciseness**: Try to use *less context* to cover *more scenarios*. Avoid adding redundant or hyper-specific templates for every single edge case. - - **Generalizability**: Prefer solutions that generalize well (e.g., use a `facet` for a column definition rather than a specific `template` for every query using that column). - - **Supported Types**: Support mutations for `template`, `facet`, and `value_search` types. -3. **Apply Mutations**: - - **Copy the Base Context**: Copy the base ContextSet file to the new destination: `autoctx/experiments//hillclimb/improved_context_vN.json`. - - **Generate New Items**: For any new context items identified in the fixing strategy (for "add" operations): - - **Invoke the `context-generation-guide` skill** to produce the final parameterized items. - - Provide the identified candidates to that skill. - - That skill will handle phrase extraction, parameterization, and constructing the valid JSON structure. - - **Validate New Items**: - - **Templates**: Run generated SQL examples via `-execute-sql` (use dummy values for placeholders) to verify syntax. - - **Others**: Cross-check table/column references against the schema via `-list-schemas`. - - **Apply Mutations**: Call the `mutate_context_set` MCP tool passing the **new** file path as `file_path` and mutations as `mutations_json` to mutate the context set. -3. **Log in State Tracking**: - - Update `autoctx/state.md` to include the output path of `improved_context_vN.json` for Loop `vN`. - ---- - -### 4. Validation & Upload Advice - -1. **Summarize Improvements**: Tell the user what was changed (e.g., added 2 facets, updated 1 template). -2. **Upload Instructions**: - - **Read Database Details**: Read `autoctx/tools.yaml` (or `db_config.yaml`) to fetch the specific project, location, and instance/cluster details for the active database. - - **Generate URL**: Call the `generate_upload_url` tool passing the extracted values to provide the direct console link to the user. - - Present the local file path to `improved_context_vN.json` and the generated console link together in a single clear message. -3. **Instruct Next Step Evaluation**: - - Instruct the user to run evaluation using the evaluating workflow on this new ContextSet to see if metrics improve. This will start Loop `N+1`. - ---- - -## Output - -Upon successful completion, the workspace must contain: -- `autoctx/experiments//hillclimb/gap_analysis_vN.md` -- `autoctx/experiments//hillclimb/improved_context_vN.json` -- Updated `autoctx/state.md` summarizing the run loop. - ---- - -## Logging State Example (`autoctx/state.md`) - -When updating `autoctx/state.md`, please append or update the `Hill-Climbing Run Log` section: - -```markdown -# Context Authoring Experiment State Tracking - -## Active Experiment: my-exp-1 - -## Hill-Climbing Run Log - -### Loop: v1 -- **Base Context**: `baseline_context.json` -- **Eval Report Path**: `autoctx/experiments/my-exp-1/eval_reports//` (containing `configs.csv`, `evals.csv`, etc.) -- **Gap Analysis**: `autoctx/experiments/my-exp-1/hillclimb/gap_analysis_v1.md` -- **Mutated Context**: `autoctx/experiments/my-exp-1/hillclimb/improved_context_v1.json` -``` - - -> [!IMPORTANT] -> **Tool Modification Rule**: Always use the `mutate_context_set` tool for all ContextSet changes. Pass mutation payloads directly to the tool — it handles all file I/O internally. **Do not read the target context set file beforehand**. +Given a golden dataset and (optionally) a base context, autonomously produce a high-quality ContextSet by iterating evaluate → analyze → mutate → re-upload until convergence. The user supplies inputs once and receives the final high-scoring ContextSet; no per-iteration approval is required. + +## Prerequisites +- A working DB connection — Toolbox MCP tools (`-list-schemas`, `-execute-sql`) must be visible to the agent throughout the run. If missing or unreachable at any point, stop and route through `context-engineering-init`; do not work around it (no bash `uvx toolbox-server invoke` fallback). +- A golden evaluation dataset (JSON, simplified format: `{id, database, nlq, golden_sql}` — see `context-engineering-evaluate`). +- ADC configured and the Gemini Data Analytics + Dataplex APIs enabled on the project (see `context-engineering-init` for preflight). +- A starting context — none, a local file, or an existing Context Store resource name. Entry flow spells out the handling per case. +- (Optional) Workspace root directory. Default: `.context-engineering/experiments//` at cwd. + +## Guidance + +Load `references/workspace.md` before any workspace interaction — it describes the internal iteration layout. + +### Entry flow +1. **Locate workspace** at `` (default `.context-engineering/experiments//`). Confirm the path with the user before creating it fresh — the directory holds every iteration's scratch state. If `state.md` exists → **resume** from the recorded iteration (see Resume rules in `references/workspace.md`). Otherwise → **fresh start**. +2. **Fresh start — seed v0.** Resolve the base context per Prerequisites: + - **None** → invoke `context-engineering-bootstrap` to generate a local file, then upload as `v0` via `upload_context_set`. + - **Local file** → upload as `v0` via `upload_context_set`. + - **Existing Context Store resource name** → treat the resource as `v0`; skip the upload and start the loop at `v1` (step 1 will seed `context_set_v1.json` via `download_context_set`). + + Record the `v0` resource name and write initial `state.md`. + +### Per-iteration loop (`vN`) +1. **Prepare `vN/`**: append the `## In-Progress: vN` marker to `state.md`, create the iteration directory, and seed `context_set_vN.json`: + - If `context_set_v(N-1).json` is on disk (normal case — the prior iteration wrote it), copy it. + - Otherwise (first iteration seeded from a caller-supplied resource name, or resuming after a crash), call `download_context_set` on the `v(N-1)` resource name recorded in `state.md`. +2. **Evaluate**: invoke `context-engineering-evaluate` with `output_dir=/vN/eval/`. Capture `job_id` and overall score. +3. **Analyze**: read `scores.csv` + `summary.csv` via `read_evaluation_result`; cluster failures by category; write findings + reasoning + planned mutations to `analysis_vN.md`. +4. **Mutate**: plan mutations from the analysis (prefer fewer general items — a facet often beats many templates). Author new items via `context-engineering-generation-guide`. Validate generated SQL via `-execute-sql` and column references via `-list-schemas`. Apply via `mutate_context_set` to `context_set_vN.json`. +5. **Upload**: call `upload_context_set` with `version="vN"`. Record the returned resource name. +6. **Update `state.md`**: replace the `## In-Progress: vN` marker with the final `### vN` entry (resource name, eval report path, analysis path, score). +7. **Check convergence** (see below). If not converged, continue to `v(N+1)`. + +### Convergence +Iterate until you don't see new improvements. When stopping, append `## Converged: ` to `state.md` so a resume knows the run terminated intentionally. + +The user can also explicitly ask to stop at any time; the current iteration completes cleanly, `## User Stop` is appended to `state.md`, and all iteration files are preserved. + +**Final output** is the version with the highest recorded score. + +## Rules +- The workspace is internal state; user-visible output is the final `cs_resource_name` and score. Do not surface intermediate iteration files as user deliverables. +- Only `Template`, `Facet`, `Value Search` types are emitted as mutations. Do not invent new item types. +- Avoid overfitting: mutations should generalize to unseen NLQs, not just fix specific failing golden pairs. +- Compose the sibling skills (`context-engineering-bootstrap`, `context-engineering-evaluate`, `context-engineering-generation-guide`); never re-implement their logic. +- Never require per-iteration user approval — the loop must run autonomously. Only pause on: convergence, error, or explicit user stop. +- Do not read the target ContextSet file directly for mutations; always use `mutate_context_set`. +- A full run is long — 10 iterations × multi-minute evaluations each can take 30–60+ minutes. Warn the user upfront and run in a session that tolerates long work. + +## Tools + +**Sibling skills:** +- `context-engineering-bootstrap` — cold-start path when no base context. +- `context-engineering-evaluate` — invoked once per iteration. +- `context-engineering-generation-guide` — produces well-formed Template / Facet / Value Search JSON. + +**MCP:** +- `upload_context_set` → `cs_resource_name` — version push per iteration. +- `download_context_set` — seed on resume. +- `mutate_context_set` — apply planned mutations. +- `read_evaluation_result` — read scored eval reports. +- `-list-schemas` (Toolbox) — validate that referenced columns exist. +- `-execute-sql` (Toolbox) — validate that generated SQL runs against the DB. + +**References:** +- `references/workspace.md` — internal workspace layout; load before any workspace interaction. diff --git a/plugin/skills/context-engineering-hillclimb/references/workspace.md b/plugin/skills/context-engineering-hillclimb/references/workspace.md new file mode 100644 index 00000000..d683a88a --- /dev/null +++ b/plugin/skills/context-engineering-hillclimb/references/workspace.md @@ -0,0 +1,85 @@ +# Hill-Climbing Workspace Layout + +The hill-climbing skill maintains an internal workspace to track iterations, evaluation reports, and mutation history. All paths are relative to `` (default `.context-engineering/experiments//`). + +## Directory structure + +``` +/ +├── state.md # metadata + per-iteration scores + notes +└── v/ # one directory per iteration (v0 = base, v1+ = mutations) + ├── context_set_v.json # ContextSet snapshot at this version + ├── analysis_v.md # findings + reasoning + mutations applied + └── eval/ # output_dir for context-engineering-evaluate + ├── eval_configs/ # generated Evalbench YAMLs + converted dataset + │ ├── db_config.yaml + │ ├── model_config.yaml + │ ├── run_config.yaml + │ ├── llmrater_config.yaml + │ └── golden_queries.json + └── eval_reports// + ├── scores.csv + └── summary.csv +``` + +## `state.md` format + +`state.md` is the single source of truth for iteration progress. It must contain: + +- **Metadata** — experiment name, workspace root, golden dataset path, DB source (from `tools.yaml`), Context Store coordinates (`project_id`, `csg_id`, `cs_id`). +- **Iteration Log** — one entry per completed iteration, in order. + +Example: + +```markdown +# Hill-Climbing Experiment: my-exp-1 + +## Metadata +- Workspace: ./.context-engineering/experiments/my-exp-1/ +- Golden dataset: ./golden.json +- DB source: my-alloydb +- Context Store: projects/

/locations//contextSetGroups/my-exp-1/contextSets/autoctx + +## Iteration Log + +### v0 (base) +- Context resource: projects/.../contextSets/autoctx@v0 +- Score: 0.42 +- Notes: baseline generated by context-engineering-bootstrap + +### v1 +- Context resource: projects/.../contextSets/autoctx@v1 +- Score: 0.58 +- Analysis: v1/analysis_v1.md +- Eval report: v1/eval/eval_reports// + +### v2 +- Context resource: projects/.../contextSets/autoctx@v2 +- Score: 0.55 +- Analysis: v2/analysis_v2.md +- Eval report: v2/eval/eval_reports// +``` + +## Markers + +`state.md` carries three markers that let a resume distinguish crash, intentional stop, and user stop: + +- `## In-Progress: vN` — written at the start of iteration N (per-iteration step 1); removed when the final `### vN` entry lands (per-iteration step 6). A leftover marker means the prior run crashed mid-iteration. +- `## Converged: ` — appended when the agent judges improvements dry. +- `## User Stop` — appended when the user explicitly asks the loop to stop. + +Completed iterations are preserved in every case — never deleted. + +## Resume rules + +On skill invocation with an existing workspace: + +1. Read `state.md`. +2. Determine the last completed iteration `N`. +3. **Check for `## In-Progress: vM`.** If present, the prior run crashed mid-iteration `M`. Delete the partial `vM/` directory (its files are not trustworthy), remove the marker, and re-run iteration `M` from step 1. +4. If `## Converged` or `## User Stop` is present, report the best-scoring iteration and stop — do not start another iteration. +5. Otherwise, start iteration `N+1`. + +## Final output + +The best-scoring iteration in the Iteration Log is the final output. diff --git a/plugin/skills/context-engineering-init/SKILL.md b/plugin/skills/context-engineering-init/SKILL.md index 6c545518..be587ba5 100644 --- a/plugin/skills/context-engineering-init/SKILL.md +++ b/plugin/skills/context-engineering-init/SKILL.md @@ -1,100 +1,104 @@ --- name: context-engineering-init -description: Orchestrates the initialization workflow for context engineering, and provides helper workflow for setting up database connections by creating or updating tools.yaml configurations. +description: Ensure the environment is ready for context-engineering work — manage the Toolbox `tools.yaml` for database connections, verify runtime and GCP setup (uv, evalbench, ADC, Dataplex/GDA APIs, IAM), and diagnose readiness failures raised by other skills. --- -> **Load the `context-engineering-workflow` skill first.** It holds the shared context this phase depends on: workspace layout, state file conventions, phase order, and safety protocol. Do not proceed with this phase without reading it. - -# Phase: Setup & Connection Configuration +# Skill: Environment & Connection Setup ## Goal -Scaffold the local `autoctx/` workspace and establish verified database connections to prepare the environment for context engineering. +Ensure the caller's environment is ready for context-engineering work: Toolbox `tools.yaml` in place with at least one verified DB source, and (when asked or when downstream failures need diagnosis) runtime + GCP readiness verified. Manage `tools.yaml` on request; run any subset of checks on request. -## Initialization Workflow +## Prerequisites +- `gcloud` CLI on PATH. +- (Optional) Target GCP project id — if not provided, use ADC's default project. +- (Optional) Existing `tools.yaml` to amend rather than overwrite. -Follow these steps when the user asks to initialize the environment: +## Guidance -1. **Confirm Working Directory:** Explicitly state the current working directory to the user. Explain that the initialization will create an `autoctx/` folder in this directory to hold `tools.yaml`, `state.md`, and `experiments/`. Ask them to confirm if this is the correct location before proceeding. -2. **Check Existing Infrastructure:** - - Check if the `autoctx/` directory exists. - - If it exists, verify if it contains valid `tools.yaml` and `state.md` files. If it appears to be an unrelated folder or corrupted, STOP and ask the user how to proceed (e.g., use a different name or overwrite). - - If it is a valid Autoctx folder and contains all items, inform the user it's already initialized. Otherwise, proceed to create missing items inside `autoctx/`. -3. **Setup Toolbox Configuration:** If `tools.yaml` is missing inside `autoctx/`, follow the primary "1. Create a New tools.yaml" workflow documented below in the **Toolbox Config Helper** section. -4. **Create State Tracker:** If `state.md` is missing inside `autoctx/`, create it with header “context authoring experiment state tracking”. -5. **Initialize Experiments Directory:** If `experiments/` is missing inside `autoctx/`, create an empty `experiments/` directory inside `autoctx/`. +Pick the flow that matches the user's intent: -## Output +### Manage `tools.yaml` (DB connections) -Upon successful completion, the workspace must contain: -- `autoctx/`: The dedicated workspace directory. - - `tools.yaml`: A structurally sound configuration file for the Toolbox MCP Server. - - `state.md`: The external state tracker for hill-climbing iterations. - - `experiments/`: The base directory prepared to store all hill-climbing run artifacts (e.g. baseline contexts, evaluation reports). +Three sub-workflows: -## Final Summary +**Create new** +1. Identify the DB type; load `references/.md` for required fields. +2. Ask the user for every required field explicitly. Do not fill in missing values. +3. Generate the YAML from the template with the user's values. +4. Save to `.context-engineering/tools.yaml`. This path is fixed — the Toolbox MCP server reads it directly; any other path won't be picked up. -Conclude by providing a succinct summary to the user: -- State whether the workspace was initialized newly or if existing files were preserved. -- Instruct the user to reload the MCP toolbox so any new database connections take effect: - - **Gemini CLI**: run `/mcp reload`. - - **Claude Code**: run `/mcp`, select `toolbox`, and select `Reconnect` — or `/quit` and relaunch Claude Code. - - **Antigravity CLI**: run `/mcp`, select `toolbox`, and select `Restart`. -- Inform them they are now ready to proceed to the next phase (e.g., the Bootstrap workflow). +**Add to existing** +1. Identify the new DB type and a unique `` name. +2. Ask for the required fields (same as Create). +3. Read the current `tools.yaml`. +4. Generate new `sources:` and `tools:` entries under the unique `` name and append them. +5. Save the updated file. ---- +**List existing** +1. Read the `tools.yaml` at the given path. If missing, tell the user. +2. Parse and list all names under `sources:`. -# Toolbox Config Helper +Validate the target source(s) standalone (Example: `uvx toolbox-server@1.4.0 --config invoke -list-schemas`) — no MCP restart needed for validation. On validation failure, drop into the checks below to diagnose (e.g., `DB source reachable`, `ADC configured`). -This section contains standalone instructions for managing the `tools.yaml` file for the GenAI Toolbox. You can execute these if the user explicitly asks to add or list database connections. +After any write, instruct the user to restart the MCP server so downstream skills see the new source: +- Gemini CLI: `/mcp reload` +- Claude Code: `/mcp` → `toolbox` → Reconnect (or `/quit` and relaunch) +- Antigravity CLI: `/mcp` → `toolbox` → Restart -## Credentials +### Verify environment (broad or scoped) -For Google Cloud databases, the system uses Application Default Credentials (ADC) and IAM Authentication. Providing a user and password is not supported. +Run any subset of the checks below. Report `PASS` or `FAIL` per check. For any `FAIL`, propose a fix and ask the user for consent before executing anything mutating (installing packages, enabling APIs, changing IAM, writing files). -When collecting information from the user, inform the user that only Application Default Credentials (ADC) are supported for authentication. They do not need to provide a username or password. +- **Broad verification** ("am I ready?") → run all checks. +- **Scoped diagnosis** (a downstream skill failed) → run only the checks whose `— required by …` line references the failing operation. -**Sample Message:** -> "I'll help you configure the database connection in `tools.yaml`. Note that the system only supports Application Default Credentials (ADC) for authentication, so you don't need to provide a username or password. Please ensure that the IAM account you are using has the required permissions to access the database. -> -> Could you please provide the following details: -> - Google Cloud Project ID: -> - Region: -> ... (other required fields based on database type)" +## Checks -## Primary Workflows +Commands in parentheses are examples — the agent may use its own approach. -### 1. Create a New `tools.yaml` +### Environment +- **`uv` installed** — required to run Toolbox and Evalbench via `uvx`. (Example: `uv --version`; install via `curl -LsSf https://astral.sh/uv/install.sh | sh` or `brew install uv`.) +- **Evalbench reachable** — required by `context-engineering-evaluate`; verifying also warms the uvx cache so the first `evaluate` run is fast. (Example: `uvx google-evalbench@1.10.0 --help`.) -1. **Identify Database Type:** Ask the user which database they want to configure: - - Cloud SQL Postgres - - Cloud SQL MySQL - - AlloyDB Postgres - - Spanner -2. **Collect Information:** Request all **Required Information** based on the templates inside this directory. Do NOT assume missing fields; ask the user for them explicitly. -3. **Generate Configuration:** Replace all placeholders with the user's provided values and generate the complete `tools.yaml` content. Save it to the target location (e.g., `autoctx/tools.yaml` for Autoctx workflows, or `tools.yaml` in the current directory for standalone use). -4. **Validate:** After saving, validate the new connection using the toolbox script, replacing `` with the actual path to the file: - `uvx toolbox-server@1.4.0 --config invoke -list-schemas` +### GCP authentication +- **ADC configured** — required by every GCP API call (Context Store, QueryData, Dataplex). (Example: `gcloud auth application-default print-access-token`; fix via `gcloud auth application-default login`.) +- **ADC quota project set** — required by Context Store; the `X-Goog-User-Project` header is derived from it. Missing → 400 on upload/download. (Example: `gcloud auth application-default print-quota-project`; fix via `gcloud auth application-default set-quota-project `.) -### 2. Add a Database to an Existing `tools.yaml` +### GCP API enablement +- **Dataplex API** (`dataplex.googleapis.com`) — required by Context Store operations (`upload_context_set`, `download_context_set`). +- **Gemini Data Analytics API** (`geminidataanalytics.googleapis.com`) — required by QueryData (used inside `context-engineering-evaluate`). -1. **Identify Database Type:** Ask the user for the type of the new database connection they wish to add. -2. **Collect Information:** Request the required information for the new connection, including a new, unique ``. -3. **Read Existing File:** Read the content of the existing `tools.yaml` from the target location. -4. **Generate and Append:** Generate the YAML snippets for the new `sources` and `tools` sections. Append these new entries to the respective sections in the existing file content. -5. **Save Configuration:** Save the updated content back to the file. -6. **Validate:** Validate only the newly added connection, replacing `` with the actual path to the file: - `uvx toolbox-server@1.4.0 --config invoke -list-schemas` +(Example: check enablement via `gcloud services list --enabled --project=`; enable via `gcloud services enable --project=`.) -### 3. List Existing Database Connections +### GCP IAM (operational probes) +- **Context Store access** — required by `upload_context_set` / `download_context_set` and by QueryData's context lookup. Probe by attempting a Dataplex Context Store read (e.g., list CSGs in ``). On 403, surface the error verbatim and ask the user to request the appropriate Context Store role from their IAM admin. +- **GDA access** — required by QueryData (used by `evaluate`). Probe by attempting a lightweight QueryData call in ``. On 403, same handling. -1. **Check and Read `tools.yaml`:** Check for the `tools.yaml` file. If it doesn't exist, inform the user. -2. **Parse and List:** Parse the YAML content and list the names of all configured data sources found under the `sources:` key limit. +### Toolbox configuration +- **`tools.yaml` present** — required by every skill that reads database schemas (`bootstrap`, `evaluate`, `hillclimb`). Check `.context-engineering/tools.yaml` (the fixed path the Toolbox MCP server reads). Missing → run the Create sub-workflow above. +- **DB source reachable** — required by any Toolbox invocation on that source. For each configured `` in `tools.yaml`, verify Toolbox can list its schemas standalone. On failure, surface the error verbatim; common causes are ADC, wrong project/region, DB IAM, or network. (Example: `uvx toolbox-server@1.4.0 --config invoke -list-schemas`.) -## Validation +## Rules +- Never execute mutating actions (`gcloud services enable`, `gcloud projects add-iam-policy-binding`, package installs, file writes) without explicit user consent — surface the exact command and let the user run it, or ask consent before running. +- ADC only for DB auth. Never write username/password into `tools.yaml`. +- `tools.yaml` always lives at `.context-engineering/tools.yaml` — do not offer or accept a different path. If the file exists, ask whether to append or overwrite. +- Do not guess DB connection details. Ask the user for every required field explicitly. -To verify that a specific database connection is configured correctly at any time, run the validation script with the target data source name: -`uvx toolbox-server@1.4.0 --config tools.yaml invoke -list-schemas` +## Credentials message (use when collecting DB info for `tools.yaml`) + +> "I'll help you configure the database connection in `tools.yaml`. The Toolbox server uses Application Default Credentials (ADC) for authentication, so you don't need to provide a username or password. Please ensure the IAM account you're using has the required permissions to access the database. +> +> Could you please provide the following details: +> - Google Cloud Project ID: +> - Region: +> - ... (other required fields based on database type)" -## Templates & Reference +## References +- `references/.md` (`alloydb-postgres.md`, `cloud-sql-mysql.md`, `cloud-sql-postgres.md`, `spanner.md`) — per-DB required fields and YAML template. -For the specific fields required for each database type and the exact YAML structure to use, refer to the templates in this directory (.../references/init/...). +## Gotchas +- **Quota project vs ADC project:** ADC infers a default project from `gcloud config`, but Context Store requires an explicit quota project via `X-Goog-User-Project`. Missing quota project → 400 from Context Store API. +- **MCP restart required for new tools.yaml sources:** Toolbox reads `tools.yaml` at MCP-server startup. Validation runs standalone, but agent visibility of new sources needs a restart. +- **AlloyDB requires `cluster` + `instance`; Cloud SQL only `instance`.** +- **Spanner uses ADC; verification fails without `gcloud auth application-default login`.** +- **Evalbench cold-cache:** first `uvx google-evalbench@1.10.0` can take minutes to download; verifying `Evalbench reachable` warms the cache so downstream `evaluate` runs are fast. diff --git a/plugin/skills/context-engineering-workflow/SKILL.md b/plugin/skills/context-engineering-workflow/SKILL.md index 086e8e22..4d0e3306 100644 --- a/plugin/skills/context-engineering-workflow/SKILL.md +++ b/plugin/skills/context-engineering-workflow/SKILL.md @@ -1,13 +1,23 @@ --- name: context-engineering-workflow -description: Context engineering for Gemini Data Analytics API's data agent developer platform tools. Generates, evaluates, and iteratively optimizes a ContextSet (Templates, Facets, Value Searches) to maximize Natural-Language-to-SQL translation accuracy. Use this skill to run the automated setup, NL-SQL pair evaluation dataset generation and expansion, bootstrapping, scoring, and optimization pipeline. For manual authoring standards and schema syntax rules, see the context-generation-guide skill. +description: Use this skill when the user is new to context engineering, asks how to get started, or is unsure which peer skill to invoke. Explains the ContextSet concepts (Templates, Facets, Value Searches), the optimization lifecycle, and routes to the right peer skill (init, dataset-generation, bootstrap, evaluate, hillclimb) based on the user's current state. --- -# Skill: Context Engineering Orchestrator +# Skill: Context Engineering Lifecycle -You are an expert context engineering agent. Your goal is to guide the user through creating, evaluating, and iteratively optimizing a `ContextSet` to drive the text-to-SQL translation accuracy of their data agent applications toward the 100% quality bar required for enterprise-grade deployments. +This skill defines the shared vocabulary, lifecycle, and safety rules used by the `context-engineering-*` peer skills. Consult it to orient before invoking a peer, or when a rule reads as "cross-cutting." -Refer to [context-generation-guide/SKILL.md](../context-generation-guide/SKILL.md) for how to edit a ContextSet. +--- + +## Shared Terminology + +A **ContextSet** is the structured knowledge blob the Gemini Data Analytics API consumes when translating a natural-language question to SQL. It contains three item types: + +* **Template** — a full NLQ → SQL mapping, generalized with placeholders (e.g., `$1`, `$2`). Teaches end-to-end query patterns. +* **Facet** — a reusable SQL fragment (e.g., a `WHERE` predicate or a specialized join), tied to specific vocabulary. Composed into generated queries dynamically. +* **Value Search** — a query that maps user-supplied terms (with typos, casing differences, or synonyms) to their exact database values. Resolves the value-linking problem. + +See [context-engineering-generation-guide](../context-engineering-generation-guide/SKILL.md) for the JSON schemas and dialect-specific authoring standards. --- @@ -18,107 +28,76 @@ To build high-performing data applications, context engineers typically follow a ```mermaid flowchart TD Start([Start]) --> Setup[Setup & Connection -Scaffolds workspace & connections] - Setup --> Prep{Has Golden Dataset?} - - Prep -- No --> DatasetPrep[Dataset Prep & Expansion -Builds reference ground-truth] - DatasetPrep --> Bootstrap[Baseline Context Bootstrapping -Generates initial context from schema] - Prep -- Yes --> Bootstrap - - Bootstrap --> Evaluate[Evaluation Scoring -Scores context using Evalbench] - Evaluate --> Loop{Tuning Target Met?} - - Loop -- No --> Hillclimb[Optimization & Hill-Climbing -Gap Analysis & Context Mutation] - Hillclimb --> Evaluate - - Loop -- Yes --> End([End - Context Deployed!]) +context-engineering-init] + Setup --> Dataset{Has Golden Dataset?} + Dataset -- No --> DatasetPrep[Dataset Prep & Expansion +context-engineering-dataset-generation] + DatasetPrep --> BaseCheck + Dataset -- Yes --> BaseCheck{Has Base Context?} + BaseCheck -- No --> Bootstrap[Baseline Bootstrap +context-engineering-bootstrap] + Bootstrap --> Optimize + BaseCheck -- Yes --> Optimize[Autonomous Optimization +context-engineering-hillclimb +_runs evaluate + analyze + mutate internally_] + Optimize --> End([End - Context Deployed]) ``` --- +## Where to Start + +If you're new, invoke the peer that matches your current state: + +* No `.context-engineering/tools.yaml` → [context-engineering-init](../context-engineering-init/SKILL.md) +* No golden dataset → [context-engineering-dataset-generation](../context-engineering-dataset-generation/SKILL.md) +* No base ContextSet → [context-engineering-bootstrap](../context-engineering-bootstrap/SKILL.md) +* Have a base ContextSet and want autonomous improvement → [context-engineering-hillclimb](../context-engineering-hillclimb/SKILL.md) +* Have a ContextSet and just want to score it → [context-engineering-evaluate](../context-engineering-evaluate/SKILL.md) + +Experienced users can skip this section and invoke any peer directly. + +--- + ## Workflow Phases, Rationales & Entry Prerequisites --- -### Setup & Connection Configuration Phase +### Setup & Connection Configuration * **Reference**: [context-engineering-init](../context-engineering-init/SKILL.md) -* **Goal**: Scaffold the local `autoctx/` workspace and establish verified database connections. -* **Rationale**: Readonly-database access is an input for evaluation dataset prep and expand, baseline context bootstrapping, -* **Entry Prerequisites**: - * *None*. +* **Goal**: Configure `.context-engineering/tools.yaml` for the Toolbox MCP server and verify runtime + GCP setup (uv, evalbench, ADC, Dataplex/GDA APIs, IAM). --- -### Evaluation Dataset Prep & Expansion Phase +### Evaluation Dataset Prep & Expansion * **Reference**: [context-engineering-dataset-generation](../context-engineering-dataset-generation/SKILL.md) -* **Mandatory Deliverables**: `evalset_environment_inputs.md`, `evalset_gen_plan.md`, `evalset_report_pair_level.md`, and `evalset_report_dataset_level.md`. -* **Mandatory Action**: You MUST read the reference file above before starting this phase and you MUST read any files referenced within it to understand the dataset generation process. -* **Goal**: Build a high-quality "golden" ground-truth dataset and associated audit reports. -* **Rationale**: A representative ground-truth dataset and formal audit trails are required to objectively measure and verify translation accuracy improvements. -* **Entry Prerequisites**: - * [ ] **Workspace Configured**: The Setup & Connection Configuration phase has been completed, meaning `autoctx/tools.yaml` is active. +* **Goal**: Build a high-quality "golden" ground-truth NLQ+SQL dataset for evaluating translation accuracy. +* **Requires**: Setup & Connection Configuration complete. --- -### Baseline Context Bootstrapping Phase +### Baseline Context Bootstrapping * **Reference**: [context-engineering-bootstrap](../context-engineering-bootstrap/SKILL.md) -* **Goal**: Deduce query concepts and generate a baseline `ContextSet` (templates, facets, value searches) directly from database schemas and metadata. -* **Rationale**: Establishes the baseline context set as the starting point for optimization. -* **Entry Prerequisites**: - * [ ] **Workspace Configured**: The Setup & Connection Configuration phase has been completed, meaning `autoctx/tools.yaml` is active. +* **Goal**: Generate a baseline `ContextSet` (Templates, Facets, Value Searches) from database schema and optional enrichment sources. +* **Requires**: Setup & Connection Configuration complete. --- -### Run Evaluation And Score +### Evaluation Scoring * **Reference**: [context-engineering-evaluate](../context-engineering-evaluate/SKILL.md) -* **Goal**: Run a structured Evalbench evaluation to score the accuracy of a specific context set and identify exact query failures. -* **Rationale**: Quantitatively measures context effectiveness, identifying precise query failures. -* **Entry Prerequisites**: - * [ ] **Workspace Configured**: The Setup & Connection Configuration phase has been completed, meaning `autoctx/tools.yaml` is active. - * [ ] **Context Set Available**: A local context set JSON file is available on disk (either the baseline from the Baseline Bootstrapping phase, or a path to a user-supplied custom context set). - * [ ] **Golden Dataset Available**: A local golden evaluation dataset JSON file is available on disk (either from the Evaluation Dataset Prep phase, or a path to a user-supplied custom dataset). - * [ ] **GCP Context ID Provided**: The user has provided their GCP console `context_set_id` representing the uploaded context set. +* **Goal**: Run Evalbench against a ContextSet + golden dataset to produce an accuracy score and per-query failure breakdown. +* **Requires**: Setup & Connection Configuration complete; a golden dataset and a ContextSet (local file or Context Store resource name) available. --- -### Optimization & Hill-Climbing Phase +### Autonomous Optimization * **Reference**: [context-engineering-hillclimb](../context-engineering-hillclimb/SKILL.md) -* **Goal**: Analyze evaluation failures to perform a Gap Analysis and apply targeted context mutations to iteratively improve performance. -* **Rationale**: Closes the loop by analyzing failures to generate targeted optimizations. -* **Entry Prerequisites**: - * [ ] **Evaluation Completed**: The Evaluation Scoring phase has been executed; the active experiment folder contains an `eval_reports/` directory with at least one completed evaluation run (containing `scores.csv` and `summary.csv`). - * [ ] **Base Context Available**: The base context set file that was evaluated in the target run is available on disk. +* **Goal**: Autonomously iterate evaluate → analyze → mutate → re-upload until improvements dry up, producing a high-scoring ContextSet with no per-iteration user approval. +* **Requires**: Setup & Connection Configuration complete; a golden dataset. The base context is optional — hillclimb invokes bootstrap when none is supplied. +* **Note**: Hillclimb runs evaluate itself every iteration; do not run Evaluation Scoring separately as a precondition. --- -## Workspace Folder Structure & Evolution - -The Autoctx workflows generate and interact with a structured workspace to maintain state and trace progress across iterations. - -### Workspace Folder Layout -* `autoctx/`: The dedicated workspace directory. - * `tools.yaml`: Configuration file for the Toolbox MCP Server. - * `state.md`: Summary of the experiment state, active experiment, and run history. - * `experiments/`: Root directory for all experiments. - * `/`: Specific experiment directory. - * `bootstrap_context.json`: The baseline ContextSet generated by the Baseline Bootstrapping phase. - * `eval_configs/`: Directory containing Evalbench configurations. - * `eval_reports/`: Directory containing evaluation output runs. - * `hillclimb/`: Directory containing hill-climbing iteration artifacts. - * `gap_analysis_vN.md`: Analysis of missing contexts at iteration `N`. - * `improved_context_vN.json`: The mutated ContextSet at iteration `N`. - -### Workspace Evolution Lifecycle -1. **Post-Initialization**: `tools.yaml`, `state.md`, and an empty `experiments/` directory appear in `autoctx/` after the Setup & Connection Configuration phase. -2. **Post-Bootstrap**: `autoctx/experiments//bootstrap_context.json` is generated by the Baseline Bootstrapping phase. -3. **Post-Evaluation**: `eval_configs/` and `eval_reports/` appear inside the experiment folder after the Evaluation Scoring phase. -4. **Post-Hill-Climbing**: `hillclimb/` appears with `gap_analysis_vN.md` and `improved_context_vN.json` after the Optimization & Hill-Climbing phase, and `state.md` is updated. -5. **Tuning Loop**: Iteratively evaluates `improved_context_vN.json` and generates `improved_context_v(N+1).json` until target accuracy is achieved. - ## Safety & Protocol * **Missing Dataset**: @@ -131,9 +110,4 @@ The Autoctx workflows generate and interact with a structured workspace to maint * Examples:`503` or `429` error, `UNAVAILABLE` or `RESOURCE_EXHAUSTED` status code. * Why: These errors are often associated with quota issues, and retrying the request immediately will not resolve the issue. For issues related to Vertex AI Resource Exhaustion, retrying at a later time is often the only solution. -* **Skill Prerequisites & Troubleshooting**: - * If you encounter any environmental, connection, or execution errors, verify that all preconditions are met. Types of preconditions: - * *Google Cloud Service APIs enablement* - * *IAM Roles & Access* - * *Database Instance Permissions* - * *Development Environment*: Application Default Credentials (ADC) and Python package manager (`uv`) \ No newline at end of file +* **Environment failures**: For any environment or connection failure (missing `tools.yaml`, MCP tools unreachable, ADC not configured, API not enabled, IAM missing), route through [context-engineering-init](../context-engineering-init/SKILL.md) to diagnose and fix. Do not invent workarounds (e.g., calling Toolbox via bash instead of MCP). \ No newline at end of file diff --git a/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py b/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py index d393f7d4..debecac1 100644 --- a/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py +++ b/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py @@ -5,6 +5,11 @@ import yaml from google.protobuf.json_format import MessageToDict +# Pinned to the QueryData autopush sandbox to match the Context Store endpoint +# used by `context_store_client`. When both APIs land on production, drop this +# and let evalbench default to the production QueryData endpoint. +_QUERY_DATA_API_ENDPOINT = "autopush-geminidataanalytics.sandbox.googleapis.com" + class BaseDBConfigGenerator(ABC): """ @@ -68,6 +73,7 @@ def generate_model_config(self, context_set_id: str) -> str: "generator": "query_data_api", "project_id": self.params.get("project"), "location": self.params.get("region") or "global", + "api_endpoint": _QUERY_DATA_API_ENDPOINT, "context": query_context_dict, } diff --git a/src/google/cloud/db_context_enrichment/main.py b/src/google/cloud/db_context_enrichment/main.py index e2f0c393..f0a49d15 100644 --- a/src/google/cloud/db_context_enrichment/main.py +++ b/src/google/cloud/db_context_enrichment/main.py @@ -133,11 +133,7 @@ def generate_upload_url( return "Error: Invalid db_engine. Must be one of 'alloydb', 'cloudsql', or 'spanner'." -# NOTE: `@mcp.tool` is intentionally NOT applied to upload_context_set / -# download_context_set. The Context Store client library ships in this -# release, but the MCP tool wrappers are held back until the Context Store -# API is stable in production. Re-add the decorator to expose these to -# agents when ready. +@mcp.tool def upload_context_set( local_file_path: str, project_id: str, @@ -177,6 +173,7 @@ def upload_context_set( return cs_resource_name +@mcp.tool def download_context_set(cs_resource_name: str, output_file_path: str) -> str: """ Download a ContextSet from the Context Store and write it to a local @@ -215,7 +212,7 @@ def mutate_context_set( - 'type': "template", "facet", or "value_search" - 'identifier' (dict): Required for "delete" and "update" to find the target item (e.g., {"nl_query": "What are all users?"}). - 'value' (dict): Required for "add" and "update". - - For "add": Must be the FULL item body. Follow the formatting guidance in the `context-generation-guide` skill to produce well-formed content. + - For "add": Must be the FULL item body. Follow the formatting guidance in the `context-engineering-generation-guide` skill to produce well-formed content. - For "update": Can be a PARTIAL body containing only the fields to change (it will be merged with the existing item). Example 'mutations_json': diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py index 036c35c8..1eab9060 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py @@ -49,6 +49,7 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "us-west1", + "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "alloydb": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py index 83767bfa..36d0e29c 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py @@ -47,6 +47,7 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "us-west1", + "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "cloud_sql_reference": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py index 93627f4b..d8f1af57 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py @@ -47,6 +47,7 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "us-west1", + "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "cloud_sql_reference": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py index 1814cedf..2b9866b9 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py @@ -44,6 +44,7 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "global", + "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "spanner_reference": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py b/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py index ff746932..8454da5e 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py @@ -152,6 +152,7 @@ def test_generate_evalbench_configs(): generator: query_data_api project_id: test-project location: us-central1 + api_endpoint: autopush-geminidataanalytics.sandbox.googleapis.com context: datasource_references: cloud_sql_reference: From d5a4ba7939a6592de0d8841ea34117058fb1ffb9 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Sun, 19 Jul 2026 17:52:18 -0700 Subject: [PATCH 02/19] wip(evals): fix workspace_post_evaluation state.md to use astana-transformation project --- .../experiments/my-alloydb-tuning-experiment/state.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md index 3d1b3bcf..57443594 100644 --- a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md +++ b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md @@ -4,12 +4,12 @@ - Workspace: ./.context-engineering/experiments/my-alloydb-tuning-experiment/ - Golden dataset: ./golden.json - DB source: my-alloydb -- Context Store: projects/cloud-db-nl2sql/locations/us-east1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx +- Context Store: projects/astana-transformation/locations/us-central1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx ## Iteration Log ### v0 (base) -- Context resource: projects/cloud-db-nl2sql/locations/us-east1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx@v0 +- Context resource: projects/astana-transformation/locations/us-central1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx@v0 - Score: 0.75 - Analysis: (baseline — no analysis) - Eval report: v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/ From 40b940b7534609d38e7b31ce499bcd1fbe5bb0c1 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 11:18:54 -0700 Subject: [PATCH 03/19] wip(evals): rename core-cujs IDs to core-cujs:*, seed v0 in hillclimb, document Context Store naming convention --- evals/core-cujs/dataset.json | 24 +++++++++---------- .../context-engineering-hillclimb/SKILL.md | 4 ++-- .../references/workspace.md | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/evals/core-cujs/dataset.json b/evals/core-cujs/dataset.json index 47ba4d0d..03710881 100644 --- a/evals/core-cujs/dataset.json +++ b/evals/core-cujs/dataset.json @@ -1,7 +1,7 @@ { "scenarios": [ { - "id": "autoctx-init", + "id": "core-cujs:init", "starting_prompt": "setup context generation for my database.", "conversation_plan": "Ask the agent to setup the auto context generation for my alloydb database. Connection information: datasource name as my-alloydb, project id is astana-transformation, region us-east4, cluster juexinw-test, instance juexinw-test-primary, database financial. Verify the agent generate a valid tools.yaml. Explicitly ask the agent to perform validation. You should terminate the conversation immediately after the agent successfully set up the files and performed the validation.", "expected_trajectory": [], @@ -10,7 +10,7 @@ "max_turns": 5 }, { - "id": "autoctx-dataset-generation", + "id": "core-cujs:dataset-generation", "starting_prompt": "Use context engineering, generate an evaluation dataset consisting of 5 questions for my alloydb database financial named golden.json, based on my seed dataset to be provided.", "conversation_plan": "Ask the agent to generate an evaluation dataset for my alloydb database financial named golden.json. The 'seed' includes the following two golden NL-SQL pair:\n\nHow many accounts who choose issuance after transaction are staying in East Bohemia region? A3 contains the data of region; 'POPLATEK PO OBRATU' represents for 'issuance after transaction'.\nSELECT COUNT(DISTINCT \"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"district\" AS \"T2\" ON \"T1\".\"district_id\" = \"T2\".\"district_id\" WHERE \"T2\".\"A3\" = 'east Bohemia' AND \"T1\".\"frequency\" = 'POPLATEK PO OBRATU'\n\nHow many accounts who have region in Prague are eligible for loans? A3 contains the data of region\nSELECT COUNT(\"T1\".\"account_id\") FROM \"account\" AS \"T1\" INNER JOIN \"loan\" AS \"T2\" ON \"T1\".\"account_id\" = \"T2\".\"account_id\" INNER JOIN \"district\" AS \"T3\" ON \"T1\".\"district_id\" = \"T3\".\"district_id\" WHERE \"T3\".\"A3\" = 'Prague'. If prompted for validating the query, reply yes to the agent. Once the initial golden dataset is generated, ask the agent to expand it with 2 additional examples in the same file. You should terminate the conversation immediately after the agent successfully generated the dataset, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], @@ -19,7 +19,7 @@ "max_turns": 5 }, { - "id": "autoctx-dataset-expansion", + "id": "core-cujs:dataset-expansion", "starting_prompt": "Use context engineering, generate an evaluation dataset consisting of 5 questions by expanding my evaluation dataset golden.json with more diverse examples using variation strategies.", "conversation_plan": "Ask the agent to expand the existing golden.json evaluation dataset by applying variation strategies to the existing pairs. The agent should apply all six strategies: paraphrasing, merging, difficulty adjustment (both upscaling and simplification), distraction injection, linguistic variation (typos and synonyms), and value substitution (which requires running SELECT DISTINCT queries on the database). When the agent presents an expansion plan, confirm it and approve all strategies. When the agent runs SQL execution validation or asks for approval of the validation reports, approve them. You should terminate the conversation immediately after the agent successfully writes the expanded pairs to golden.json, even if the agent prompts you to perform evaluation or take next steps. At the end of the conversation, verify that the agent produced the following output files: golden.json, evalset_expansion_report_pair_level.md, evalset_expansion_report_dataset_level.md.", "expected_trajectory": [], @@ -28,7 +28,7 @@ "max_turns": 8 }, { - "id": "autoctx-bootstrap", + "id": "core-cujs:bootstrap", "starting_prompt": "create initial context set for my alloydb database", "conversation_plan": "Ask the agent to create initial context set for my alloydb database. Need to have at least 2 templates and 2 facets. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully created the context set, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], @@ -37,17 +37,17 @@ "max_turns": 5 }, { - "id": "autoctx-evaluate", - "starting_prompt": "Evaluate golden.json against my local context set at bootstrap_context.json. Upload it with project_id=astana-transformation, csg_id=my-alloydb-tuning-experiment, cs_id=autoctx, version=v0.", - "conversation_plan": "The agent should upload the local ContextSet using the exact coordinates in the starting prompt (no substitution), run the evaluation, and report scored results. Terminate after the results are presented.", + "id": "core-cujs:evaluate", + "starting_prompt": "evaluate my context set against the golden dataset. If you need any details, stop and ask before proceeding.", + "conversation_plan": "Ask the agent to evaluate the golden dataset at ./golden.json against the local ContextSet at ./bootstrap_context.json. If prompted for Context Store coordinates, use project_id=astana-transformation, csg_id=my-alloydb-tuning-experiment, cs_id=autoctx, version=v0. Terminate immediately after the agent presents evaluation results, even if it prompts to continue.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_bootstrap/", "max_turns": 5 }, { - "id": "autoctx-hillclimb-continue", - "starting_prompt": "improve the context set based on eval failure", + "id": "core-cujs:hillclimb-continue", + "starting_prompt": "improve the context set based on eval failure. If you need any details, stop and ask before proceeding.", "conversation_plan": "Ask the agent to perform the hillclimbing to improve the context set based on eval failure. The workspace already contains an in-progress experiment at .context-engineering/experiments/my-alloydb-tuning-experiment/ with a v0 baseline. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully performed the hillclimbing task, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], "kind": "agents", @@ -55,9 +55,9 @@ "max_turns": 5 }, { - "id": "autoctx-hillclimb-fresh", - "starting_prompt": "improve my context set from scratch using the golden dataset", - "conversation_plan": "Ask the agent to run hillclimb using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. If prompted for experiment name, use 'my-alloydb-fresh-experiment'. If prompted for Context Store coordinates, use csg_id 'my-alloydb-fresh-experiment', cs_id 'autoctx', version 'v0'. Terminate immediately after the agent successfully performs the hillclimbing task, even if the agent prompts you to continue.", + "id": "core-cujs:hillclimb-fresh", + "starting_prompt": "improve my context set from scratch using the golden dataset. If you need any details, stop and ask before proceeding.", + "conversation_plan": "Ask the agent to run hillclimb using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. Terminate immediately after the agent successfully performs the hillclimbing task, even if the agent prompts you to continue.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_fresh_hillclimb/", diff --git a/plugin/skills/context-engineering-hillclimb/SKILL.md b/plugin/skills/context-engineering-hillclimb/SKILL.md index 7cf5ff48..904684b1 100644 --- a/plugin/skills/context-engineering-hillclimb/SKILL.md +++ b/plugin/skills/context-engineering-hillclimb/SKILL.md @@ -26,9 +26,9 @@ Load `references/workspace.md` before any workspace interaction — it describes 2. **Fresh start — seed v0.** Resolve the base context per Prerequisites: - **None** → invoke `context-engineering-bootstrap` to generate a local file, then upload as `v0` via `upload_context_set`. - **Local file** → upload as `v0` via `upload_context_set`. - - **Existing Context Store resource name** → treat the resource as `v0`; skip the upload and start the loop at `v1` (step 1 will seed `context_set_v1.json` via `download_context_set`). + - **Existing Context Store resource name** → download the body to `v0/context_set_v0.json` via `download_context_set`; treat the resource as `v0` (no re-upload). - Record the `v0` resource name and write initial `state.md`. + Record the `v0` resource name and write initial `state.md`. Then seed `v0/`: ensure `context_set_v0.json` is on disk, evaluate into `v0/eval/`, record the baseline score. Iteration loop starts at `v1`. ### Per-iteration loop (`vN`) 1. **Prepare `vN/`**: append the `## In-Progress: vN` marker to `state.md`, create the iteration directory, and seed `context_set_vN.json`: diff --git a/plugin/skills/context-engineering-hillclimb/references/workspace.md b/plugin/skills/context-engineering-hillclimb/references/workspace.md index d683a88a..543a4bfc 100644 --- a/plugin/skills/context-engineering-hillclimb/references/workspace.md +++ b/plugin/skills/context-engineering-hillclimb/references/workspace.md @@ -26,7 +26,7 @@ The hill-climbing skill maintains an internal workspace to track iterations, eva `state.md` is the single source of truth for iteration progress. It must contain: -- **Metadata** — experiment name, workspace root, golden dataset path, DB source (from `tools.yaml`), Context Store coordinates (`project_id`, `csg_id`, `cs_id`). +- **Metadata** — experiment name, workspace root, golden dataset path, DB source (from `tools.yaml`), Context Store coordinates. By convention: `csg_id = experiment_name`, `cs_id = "autoctx"`, `version = vN` per iteration. - **Iteration Log** — one entry per completed iteration, in order. Example: From 564328dc9cfb0f143d300c6dda8207ffa6cbd23e Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 12:46:11 -0700 Subject: [PATCH 04/19] fix(ci): remove duplicate check_eval_outputs invocation from run_gemini_cli.sh --- .ci/run_gemini_cli.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ci/run_gemini_cli.sh b/.ci/run_gemini_cli.sh index a4ea9c57..8c341f84 100755 --- a/.ci/run_gemini_cli.sh +++ b/.ci/run_gemini_cli.sh @@ -64,7 +64,4 @@ uv run --no-sync --project /evalbench python /evalbench/evalbench/evalbench.py - echo "Validating mandatory output files for ${SUITE}..." python3 /workspace/.ci/check_eval_outputs.py "${WORK_DIR}/${SUITE}" "${WORK_DIR}/${SUITE}/dataset.json" -echo "Validating mandatory output files for ${SUITE}..." -python3 /workspace/.ci/check_eval_outputs.py "${WORK_DIR}/${SUITE}" "${WORK_DIR}/${SUITE}/dataset.json" - touch "/workspace/EVAL_RAN_${SUT}-${SUITE}" From 217db3a842327753bb1d0f00db83c0db96b84745 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 13:10:18 -0700 Subject: [PATCH 05/19] chore(ci): remove cloudbuild-oneoff.yaml (unused; local runs use uvx directly) --- .ci/cloudbuild-oneoff.yaml | 62 -------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 .ci/cloudbuild-oneoff.yaml diff --git a/.ci/cloudbuild-oneoff.yaml b/.ci/cloudbuild-oneoff.yaml deleted file mode 100644 index 27916abd..00000000 --- a/.ci/cloudbuild-oneoff.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# One-off Cloud Build config for running a single evalbench scenario against -# the Gemini CLI SUT. Bypasses PR-label gating, preflight, and pass-rate -# checks — intended for ad-hoc developer runs. -# -# Usage: -# gcloud builds submit --config=.ci/cloudbuild-oneoff.yaml \ -# --substitutions=_SUITE=core-cujs,_SCENARIOS=autoctx-init \ -# --project=cloud-db-nl2sql -steps: - # --- Build extension (same as the main pipeline) --- - - id: build-extension - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}' - entrypoint: 'bash' - args: - - '-c' - - | - set -e - apt-get update && apt-get install -y binutils - chmod +x .ci/build.sh - .ci/build.sh - - # --- Marker + release-version files (replacing preflight for manual runs) --- - - id: preflight-oneoff - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}' - entrypoint: 'bash' - waitFor: ['-'] - args: - - '-c' - - | - set -e - touch /workspace/SHOULD_RUN_GEMINI_CLI - printf 'manual-oneoff-%s' "$BUILD_ID" > /workspace/RELEASE_VERSION - - # --- The scoped eval --- - - id: eval-gemini-cli-oneoff - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:${_EVALBENCH_IMAGE_TAG}' - entrypoint: 'bash' - secretEnv: ['ADC_KEY'] - waitFor: ['build-extension', 'preflight-oneoff'] - env: - - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' - - 'EVAL_REPORTING_PROJECT=$_EVAL_REPORTING_PROJECT' - - 'EVAL_SCENARIOS=$_SCENARIOS' - args: ['.ci/run_gemini_cli.sh', '$_SUITE'] - -availableSecrets: - secretManager: - - versionName: projects/cloud-db-nl2sql/secrets/context-engineering-agent-adc-credentials/versions/latest - env: 'ADC_KEY' - -serviceAccount: 'projects/cloud-db-nl2sql/serviceAccounts/crema-eval@cloud-db-nl2sql.iam.gserviceaccount.com' - -substitutions: - _EVALBENCH_IMAGE_TAG: 'latest' - _PROJECT_LOCATION: 'global' - _EVAL_REPORTING_PROJECT: 'cloud-db-nl2sql' - _SUITE: 'core-cujs' - _SCENARIOS: 'autoctx-init' - -options: - defaultLogsBucketBehavior: REGIONAL_USER_OWNED_BUCKET From 3d6c976e0e9f29045886783476d13fdfeaa91134 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 14:24:20 -0700 Subject: [PATCH 06/19] chore(plugin): use placeholder substitution for extension vars; remove empty GEMINI.md context file --- .ci/build.sh | 3 +-- .ci/run_gemini_cli.sh | 14 +++++++------- dev-plugin/gemini-extension.json | 3 +-- evals/model_configs/gemini_cli_model.yaml | 4 +++- gemini-extension.json | 1 - plugin/GEMINI.md | 0 plugin/gemini-extension.json | 1 - 7 files changed, 12 insertions(+), 14 deletions(-) delete mode 100644 plugin/GEMINI.md diff --git a/.ci/build.sh b/.ci/build.sh index da044d0e..483b66f2 100644 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -58,9 +58,8 @@ mv evalbench staging/skills/autoctx-evaluate/scripts/ BINARY_NAME="\${extensionPath}/google-cloud-db-context-engineering" TOOLBOX_NAME="\${extensionPath}/skills/autoctx-init/scripts/toolbox" -jq ".contextFileName = \"GEMINI.md\" | .mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \".context-engineering/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json +jq ".mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \".context-engineering/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json -cp plugin/GEMINI.md staging/ cp LICENSE staging/ # Create tarball diff --git a/.ci/run_gemini_cli.sh b/.ci/run_gemini_cli.sh index 8c341f84..a8d56936 100755 --- a/.ci/run_gemini_cli.sh +++ b/.ci/run_gemini_cli.sh @@ -32,15 +32,15 @@ cp -r "/workspace/evals/${SUITE}" "${WORK_DIR}/" cp -r "/workspace/evals/model_configs" "${WORK_DIR}/" cd "${WORK_DIR}" -# Point the Gemini CLI extension installer at the locally built extension -# instead of pulling from GitHub. -sed -i 's|https://github.com/GoogleCloudPlatform/db-context-enrichment|/workspace/staging|g' "model_configs/gemini_cli_model.yaml" +# Point the Gemini CLI extension installer at the locally built extension. +sed -i "s||/workspace/staging|g" "model_configs/gemini_cli_model.yaml" -# Inject Vertex project/location into the CLI env block so the extension can -# talk to the right GCP project without the values being committed to the repo. +# Substitute Vertex project/location placeholders in the CLI env block so the +# extension can talk to the right GCP project without the values being +# committed to the repo. sed -i \ - -e "/^ GEMINI_MODEL:/a\\ GOOGLE_CLOUD_PROJECT: \"${EVAL_GCP_PROJECT_ID}\"" \ - -e "/^ GEMINI_MODEL:/a\\ GOOGLE_CLOUD_LOCATION: \"${EVAL_GCP_PROJECT_REGION}\"" \ + -e "s||${EVAL_GCP_PROJECT_ID}|g" \ + -e "s||${EVAL_GCP_PROJECT_REGION}|g" \ "model_configs/gemini_cli_model.yaml" # Append the runtime release_version and resolve the reporting-project diff --git a/dev-plugin/gemini-extension.json b/dev-plugin/gemini-extension.json index 23c55e6e..b8662943 100644 --- a/dev-plugin/gemini-extension.json +++ b/dev-plugin/gemini-extension.json @@ -1,14 +1,13 @@ { "name": "google-cloud-db-context-engineering-dev", "version": "0.0.0-dev", - "contextFileName": "../plugin/GEMINI.md", "mcpServers": { "db-context-engineering": { "command": "uv", "args": [ "run", "--project", - "", + "/Users/juexinw/Workspace/nla/db-context-enrichment", "google-cloud-db-context-engineering" ] }, diff --git a/evals/model_configs/gemini_cli_model.yaml b/evals/model_configs/gemini_cli_model.yaml index 71910e1e..a1d4aa50 100644 --- a/evals/model_configs/gemini_cli_model.yaml +++ b/evals/model_configs/gemini_cli_model.yaml @@ -3,9 +3,11 @@ generator: gemini_cli env: GOOGLE_GENAI_USE_VERTEXAI: "true" GEMINI_MODEL: "gemini-3-flash-preview" + GOOGLE_CLOUD_PROJECT: "" + GOOGLE_CLOUD_LOCATION: "" setup: extensions: - "https://github.com/GoogleCloudPlatform/db-context-enrichment": + "": settings: { } diff --git a/gemini-extension.json b/gemini-extension.json index c39cc5b1..1ad261a4 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,7 +1,6 @@ { "name": "google-cloud-db-context-engineering", "version": "0.6.0", - "contextFileName": "./plugin/GEMINI.md", "mcpServers": { "db-context-engineering": { "command": "uvx", diff --git a/plugin/GEMINI.md b/plugin/GEMINI.md deleted file mode 100644 index e69de29b..00000000 diff --git a/plugin/gemini-extension.json b/plugin/gemini-extension.json index f283a6d2..77e39f67 100644 --- a/plugin/gemini-extension.json +++ b/plugin/gemini-extension.json @@ -1,7 +1,6 @@ { "name": "google-cloud-db-context-engineering", "version": "0.6.0", - "contextFileName": "./GEMINI.md", "mcpServers": { "mcp_db_context_engineering": { "command": "uv", From b7b756506c22500d11e20726d692a6725f485110 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 14:33:40 -0700 Subject: [PATCH 07/19] chore(evals): remove duplicate autoctx/ tools.yaml fixtures in eval-dataset-gen (already renamed to .context-engineering/) --- .../autoctx/experiments/.gitkeep | 0 .../autoctx/state.md | 1 - .../autoctx/tools.yaml | 33 ------------------- .../autoctx/experiments/.gitkeep | 0 .../autoctx/state.md | 1 - .../autoctx/tools.yaml | 33 ------------------- .../autoctx/experiments/.gitkeep | 0 .../autoctx/state.md | 1 - .../autoctx/tools.yaml | 33 ------------------- .../autoctx/experiments/.gitkeep | 0 .../autoctx/state.md | 1 - .../autoctx/tools.yaml | 33 ------------------- .../autoctx/experiments/.gitkeep | 0 .../autoctx/state.md | 1 - .../autoctx/tools.yaml | 33 ------------------- 15 files changed, 170 deletions(-) delete mode 100644 evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/experiments/.gitkeep delete mode 100644 evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/state.md delete mode 100644 evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/tools.yaml delete mode 100644 evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/experiments/.gitkeep delete mode 100644 evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/state.md delete mode 100644 evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/tools.yaml delete mode 100644 evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/experiments/.gitkeep delete mode 100644 evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/state.md delete mode 100644 evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/tools.yaml delete mode 100644 evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/experiments/.gitkeep delete mode 100644 evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/state.md delete mode 100644 evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/tools.yaml delete mode 100644 evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/experiments/.gitkeep delete mode 100644 evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/state.md delete mode 100644 evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/tools.yaml diff --git a/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/experiments/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/state.md b/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/tools.yaml deleted file mode 100644 index cac5a034..00000000 --- a/evals/eval-dataset-gen/workspace_grounding_github_code/autoctx/tools.yaml +++ /dev/null @@ -1,33 +0,0 @@ -kind: source -name: my-alloydb -type: alloydb-postgres -project: cloud-db-nl2sql -region: us-central1 -cluster: whaoyu-test -instance: whaoyu-test-primary -database: search ---- -kind: tool -name: my-alloydb-list-schemas -type: postgres-list-tables -source: my-alloydb -description: | - Use this tool to list tables and their schemas in the my-alloydb database. - - Progressive Schema Discovery (Recommended): - 1) Fetch structure first (output_format='simple'), - 2) Go deep on specific parts if interested, - 3) Use batching if info is too large. - - Scope: - - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. - - Behavior: - - Omit 'table_names' to fetch all tables. - - Omit 'output_format' for detailed schema (default). ---- -kind: tool -name: my-alloydb-execute-sql -type: postgres-execute-sql -source: my-alloydb -description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/experiments/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/state.md b/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/tools.yaml deleted file mode 100644 index b01c55fd..00000000 --- a/evals/eval-dataset-gen/workspace_grounding_local_code/autoctx/tools.yaml +++ /dev/null @@ -1,33 +0,0 @@ -kind: source -name: my-alloydb -type: alloydb-postgres -project: cloud-db-nl2sql -region: us-central1 -cluster: whaoyu-test -instance: whaoyu-test-primary -database: db_hr ---- -kind: tool -name: my-alloydb-list-schemas -type: postgres-list-tables -source: my-alloydb -description: | - Use this tool to list tables and their schemas in the my-alloydb database. - - Progressive Schema Discovery (Recommended): - 1) Fetch structure first (output_format='simple'), - 2) Go deep on specific parts if interested, - 3) Use batching if info is too large. - - Scope: - - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. - - Behavior: - - Omit 'table_names' to fetch all tables. - - Omit 'output_format' for detailed schema (default). ---- -kind: tool -name: my-alloydb-execute-sql -type: postgres-execute-sql -source: my-alloydb -description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/experiments/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/state.md b/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/tools.yaml deleted file mode 100644 index 2624bc1e..00000000 --- a/evals/eval-dataset-gen/workspace_grounding_querylog/autoctx/tools.yaml +++ /dev/null @@ -1,33 +0,0 @@ -kind: source -name: my-alloydb -type: alloydb-postgres -project: cloud-db-nl2sql -region: us-central1 -cluster: whaoyu-test -instance: whaoyu-test-primary -database: financial ---- -kind: tool -name: my-alloydb-list-schemas -type: postgres-list-tables -source: my-alloydb -description: | - Use this tool to list tables and their schemas in the my-alloydb database. - - Progressive Schema Discovery (Recommended): - 1) Fetch structure first (output_format='simple'), - 2) Go deep on specific parts if interested, - 3) Use batching if info is too large. - - Scope: - - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. - - Behavior: - - Omit 'table_names' to fetch all tables. - - Omit 'output_format' for detailed schema (default). ---- -kind: tool -name: my-alloydb-execute-sql -type: postgres-execute-sql -source: my-alloydb -description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/experiments/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/state.md b/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/tools.yaml deleted file mode 100644 index 30cbe22e..00000000 --- a/evals/eval-dataset-gen/workspace_resolve_biz_rule_shift/autoctx/tools.yaml +++ /dev/null @@ -1,33 +0,0 @@ -kind: source -name: my-alloydb -type: alloydb-postgres -project: cloud-db-nl2sql -region: us-central1 -cluster: whaoyu-test -instance: whaoyu-test-primary -database: petstore ---- -kind: tool -name: my-alloydb-list-schemas -type: postgres-list-tables -source: my-alloydb -description: | - Use this tool to list tables and their schemas in the my-alloydb database. - - Progressive Schema Discovery (Recommended): - 1) Fetch structure first (output_format='simple'), - 2) Go deep on specific parts if interested, - 3) Use batching if info is too large. - - Scope: - - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. - - Behavior: - - Omit 'table_names' to fetch all tables. - - Omit 'output_format' for detailed schema (default). ---- -kind: tool -name: my-alloydb-execute-sql -type: postgres-execute-sql -source: my-alloydb -description: Use this tool to execute SQL statements against the my-alloydb database. diff --git a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/experiments/.gitkeep b/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/experiments/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/state.md b/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/state.md deleted file mode 100644 index d5ca461c..00000000 --- a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/state.md +++ /dev/null @@ -1 +0,0 @@ -# context authoring experiment state tracking diff --git a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/tools.yaml b/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/tools.yaml deleted file mode 100644 index 6cc51853..00000000 --- a/evals/eval-dataset-gen/workspace_resolve_cryptic_schema/autoctx/tools.yaml +++ /dev/null @@ -1,33 +0,0 @@ -kind: source -name: my-alloydb -type: alloydb-postgres -project: cloud-db-nl2sql -region: us-central1 -cluster: whaoyu-test -instance: whaoyu-test-primary -database: db_ecommerce_cryptic ---- -kind: tool -name: my-alloydb-list-schemas -type: postgres-list-tables -source: my-alloydb -description: | - Use this tool to list tables and their schemas in the my-alloydb database. - - Progressive Schema Discovery (Recommended): - 1) Fetch structure first (output_format='simple'), - 2) Go deep on specific parts if interested, - 3) Use batching if info is too large. - - Scope: - - The tool can fetch system/extension schemas. Agents should ignore them and focus on user data. - - Behavior: - - Omit 'table_names' to fetch all tables. - - Omit 'output_format' for detailed schema (default). ---- -kind: tool -name: my-alloydb-execute-sql -type: postgres-execute-sql -source: my-alloydb -description: Use this tool to execute SQL statements against the my-alloydb database. From 198ef10f1069da8740f37d436e5b4bcf1d230fbd Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 14:59:02 -0700 Subject: [PATCH 08/19] chore(deps): bump evalbench_version to 1.10.0 to match SKILL.md pin --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 45f1b5ce..c04c93c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ dev = [ [tool.db-context-engineering] toolbox_version = "1.4.0" -evalbench_version = "1.9.0" +evalbench_version = "1.10.0" [tool.ruff] line-length = 88 From f0b1f80a83d6552b6bf83f3b7e2bfabd735275ba Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 15:00:43 -0700 Subject: [PATCH 09/19] chore(evals): bump gemini_model execs_per_minute to 70 for faster runs --- evals/model_configs/gemini_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/model_configs/gemini_model.yaml b/evals/model_configs/gemini_model.yaml index 4bb440a0..9488e69c 100644 --- a/evals/model_configs/gemini_model.yaml +++ b/evals/model_configs/gemini_model.yaml @@ -1,4 +1,4 @@ generator: gcp_vertex_gemini vertex_model: gemini-3-flash-preview base_prompt: "" -execs_per_minute: 5 +execs_per_minute: 70 From 68461adc422db5949cc079b2ceb7f277fefe872f Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Mon, 20 Jul 2026 17:13:55 -0700 Subject: [PATCH 10/19] chore(plugin): remove broken CLAUDE.md symlinks (targets were deleted with GEMINI.md) --- dev-plugin/plugin/CLAUDE.md | 1 - plugin/CLAUDE.md | 1 - 2 files changed, 2 deletions(-) delete mode 120000 dev-plugin/plugin/CLAUDE.md delete mode 120000 plugin/CLAUDE.md diff --git a/dev-plugin/plugin/CLAUDE.md b/dev-plugin/plugin/CLAUDE.md deleted file mode 120000 index bd14241d..00000000 --- a/dev-plugin/plugin/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -../../plugin/GEMINI.md \ No newline at end of file diff --git a/plugin/CLAUDE.md b/plugin/CLAUDE.md deleted file mode 120000 index e3c5a92d..00000000 --- a/plugin/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -GEMINI.md \ No newline at end of file From 972e58214722854b20b3e4d3a8b7f48d7e8b447d Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Tue, 21 Jul 2026 13:38:15 -0700 Subject: [PATCH 11/19] chore(core-cujs): rename experiment to crema-eval-experiment and revert fixtures to cloud-db-nl2sql --- evals/core-cujs/dataset.json | 10 +++++----- .../.context-engineering/tools.yaml | 8 ++++---- .../.context-engineering/tools.yaml | 8 ++++---- .../.context-engineering/tools.yaml | 8 ++++---- .../experiments/crema-eval-experiment/state.md | 16 ++++++++++++++++ .../v0/context_set_v0.json | 0 .../v0/eval/eval_configs/db_config.yaml | 0 .../v0/eval/eval_configs/golden_queries.json | 0 .../v0/eval/eval_configs/llmrater_config.yaml | 0 .../v0/eval/eval_configs/model_config.yaml | 0 .../v0/eval/eval_configs/run_config.yaml | 10 +++++----- .../configs.csv | 10 +++++----- .../evals.csv | 0 .../scores.csv | 0 .../summary.csv | 0 .../my-alloydb-tuning-experiment/state.md | 16 ---------------- .../.context-engineering/tools.yaml | 8 ++++---- .../.context-engineering/tools.yaml | 8 ++++---- 18 files changed, 51 insertions(+), 51 deletions(-) create mode 100644 evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/state.md rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/context_set_v0.json (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_configs/db_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_configs/golden_queries.json (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_configs/llmrater_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_configs/model_config.yaml (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_configs/run_config.yaml (68%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv (86%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv (100%) rename evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/{my-alloydb-tuning-experiment => crema-eval-experiment}/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv (100%) delete mode 100644 evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md diff --git a/evals/core-cujs/dataset.json b/evals/core-cujs/dataset.json index 03710881..131c886f 100644 --- a/evals/core-cujs/dataset.json +++ b/evals/core-cujs/dataset.json @@ -3,7 +3,7 @@ { "id": "core-cujs:init", "starting_prompt": "setup context generation for my database.", - "conversation_plan": "Ask the agent to setup the auto context generation for my alloydb database. Connection information: datasource name as my-alloydb, project id is astana-transformation, region us-east4, cluster juexinw-test, instance juexinw-test-primary, database financial. Verify the agent generate a valid tools.yaml. Explicitly ask the agent to perform validation. You should terminate the conversation immediately after the agent successfully set up the files and performed the validation.", + "conversation_plan": "Ask the agent to setup the auto context generation for my alloydb database. Connection information: datasource name as my-alloydb, project id is cloud-db-nl2sql, region us-central1, cluster whaoyu-test, instance whaoyu-test-primary, database financial. Verify the agent generate a valid tools.yaml. Explicitly ask the agent to perform validation. You should terminate the conversation immediately after the agent successfully set up the files and performed the validation.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_empty/", @@ -30,7 +30,7 @@ { "id": "core-cujs:bootstrap", "starting_prompt": "create initial context set for my alloydb database", - "conversation_plan": "Ask the agent to create initial context set for my alloydb database. Need to have at least 2 templates and 2 facets. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully created the context set, even if the agent prompts you to perform evaluation or take next steps.", + "conversation_plan": "Ask the agent to create initial context set for my alloydb database. Need to have at least 2 templates and 2 facets. If prompted for experiment name, use 'crema-eval-experiment'. You should terminate the conversation immediately after the agent successfully created the context set, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_expansion/", @@ -39,7 +39,7 @@ { "id": "core-cujs:evaluate", "starting_prompt": "evaluate my context set against the golden dataset. If you need any details, stop and ask before proceeding.", - "conversation_plan": "Ask the agent to evaluate the golden dataset at ./golden.json against the local ContextSet at ./bootstrap_context.json. If prompted for Context Store coordinates, use project_id=astana-transformation, csg_id=my-alloydb-tuning-experiment, cs_id=autoctx, version=v0. Terminate immediately after the agent presents evaluation results, even if it prompts to continue.", + "conversation_plan": "Ask the agent to evaluate the golden dataset at ./golden.json against the local ContextSet at ./bootstrap_context.json. If prompted for Context Store coordinates, use project_id=cloud-db-nl2sql, csg_id=crema-eval-experiment, cs_id=autoctx, version=v0. Terminate immediately after the agent presents evaluation results, even if it prompts to continue.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_bootstrap/", @@ -48,7 +48,7 @@ { "id": "core-cujs:hillclimb-continue", "starting_prompt": "improve the context set based on eval failure. If you need any details, stop and ask before proceeding.", - "conversation_plan": "Ask the agent to perform the hillclimbing to improve the context set based on eval failure. The workspace already contains an in-progress experiment at .context-engineering/experiments/my-alloydb-tuning-experiment/ with a v0 baseline. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. You should terminate the conversation immediately after the agent successfully performed the hillclimbing task, even if the agent prompts you to perform evaluation or take next steps.", + "conversation_plan": "Ask the agent to perform the hillclimbing to improve the context set based on eval failure. The workspace already contains an in-progress experiment at .context-engineering/experiments/crema-eval-experiment/ with a v0 baseline. If prompted for experiment name, use 'crema-eval-experiment'. You should terminate the conversation immediately after the agent successfully performed the hillclimbing task, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_post_evaluation/", @@ -57,7 +57,7 @@ { "id": "core-cujs:hillclimb-fresh", "starting_prompt": "improve my context set from scratch using the golden dataset. If you need any details, stop and ask before proceeding.", - "conversation_plan": "Ask the agent to run hillclimb using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. If prompted for experiment name, use 'my-alloydb-tuning-experiment'. Terminate immediately after the agent successfully performs the hillclimbing task, even if the agent prompts you to continue.", + "conversation_plan": "Ask the agent to run hillclimb using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. If prompted for experiment name, use 'crema-eval-experiment'. Terminate immediately after the agent successfully performs the hillclimbing task, even if the agent prompts you to continue.", "expected_trajectory": [], "kind": "agents", "work_dir": "workspace_fresh_hillclimb/", diff --git a/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml b/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml index c66f2491..2624bc1e 100644 --- a/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml +++ b/evals/core-cujs/workspace_fresh_hillclimb/.context-engineering/tools.yaml @@ -1,10 +1,10 @@ kind: source name: my-alloydb type: alloydb-postgres -project: astana-transformation -region: us-east4 -cluster: juexinw-test -instance: juexinw-test-primary +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary database: financial --- kind: tool diff --git a/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml index c66f2491..2624bc1e 100644 --- a/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml +++ b/evals/core-cujs/workspace_post_bootstrap/.context-engineering/tools.yaml @@ -1,10 +1,10 @@ kind: source name: my-alloydb type: alloydb-postgres -project: astana-transformation -region: us-east4 -cluster: juexinw-test -instance: juexinw-test-primary +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary database: financial --- kind: tool diff --git a/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml index c66f2491..2624bc1e 100644 --- a/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml +++ b/evals/core-cujs/workspace_post_dataset_generation/.context-engineering/tools.yaml @@ -1,10 +1,10 @@ kind: source name: my-alloydb type: alloydb-postgres -project: astana-transformation -region: us-east4 -cluster: juexinw-test -instance: juexinw-test-primary +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary database: financial --- kind: tool diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/state.md b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/state.md new file mode 100644 index 00000000..8b30fa3f --- /dev/null +++ b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/state.md @@ -0,0 +1,16 @@ +# Hill-Climbing Experiment: crema-eval-experiment + +## Metadata +- Workspace: ./.context-engineering/experiments/crema-eval-experiment/ +- Golden dataset: ./golden.json +- DB source: my-alloydb +- Context Store: projects/cloud-db-nl2sql/locations/us-central1/contextSetGroups/crema-eval-experiment/contextSets/autoctx + +## Iteration Log + +### v0 (base) +- Context resource: projects/cloud-db-nl2sql/locations/us-central1/contextSetGroups/crema-eval-experiment/contextSets/autoctx@v0 +- Score: 0.75 +- Analysis: (baseline — no analysis) +- Eval report: v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/ +- Notes: baseline uploaded from bootstrap_context.json diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/context_set_v0.json b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/context_set_v0.json similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/context_set_v0.json rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/context_set_v0.json diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/db_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/db_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/db_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/db_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/golden_queries.json b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/golden_queries.json similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/golden_queries.json rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/golden_queries.json diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/llmrater_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/llmrater_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/llmrater_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/llmrater_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/model_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/model_config.yaml similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/model_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/model_config.yaml diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/run_config.yaml b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/run_config.yaml similarity index 68% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/run_config.yaml rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/run_config.yaml index 4aba9c22..c0686c93 100644 --- a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_configs/run_config.yaml +++ b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_configs/run_config.yaml @@ -1,10 +1,10 @@ ############################################################ ### Dataset / Eval Items ############################################################ -dataset_config: autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/golden_queries.json +dataset_config: autoctx/experiments/crema-eval-experiment/eval_configs/golden_queries.json dataset_format: evalbench-standard-format database_configs: - - autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/db_config.yaml + - autoctx/experiments/crema-eval-experiment/eval_configs/db_config.yaml dialect: postgres # DB connection mapping query_types: - dql @@ -12,7 +12,7 @@ query_types: ############################################################ ### Prompt and Generation Modules ############################################################ -model_config: autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/model_config.yaml +model_config: autoctx/experiments/crema-eval-experiment/eval_configs/model_config.yaml prompt_generator: 'NOOPGenerator' ############################################################ @@ -27,11 +27,11 @@ runners: ############################################################ scorers: llmrater: - model_config: autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/llmrater_config.yaml + model_config: autoctx/experiments/crema-eval-experiment/eval_configs/llmrater_config.yaml ############################################################ ### Reporting Related Configs ############################################################ reporting: csv: - output_directory: 'autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/' + output_directory: 'autoctx/experiments/crema-eval-experiment/eval_reports/' diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv similarity index 86% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv index 90fb475e..301ad6b3 100644 --- a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv +++ b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/configs.csv @@ -1,15 +1,15 @@ job_id,run_time,config,value -e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.dataset_config,autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/golden_queries.json +e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.dataset_config,autoctx/experiments/crema-eval-experiment/eval_configs/golden_queries.json e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.dataset_format,evalbench-standard-format -e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.database_configs,['autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/db_config.yaml'] +e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.database_configs,['autoctx/experiments/crema-eval-experiment/eval_configs/db_config.yaml'] e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.dialect,postgres e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.query_types,['dql'] -e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.model_config,autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/model_config.yaml +e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.model_config,autoctx/experiments/crema-eval-experiment/eval_configs/model_config.yaml e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.prompt_generator,NOOPGenerator e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.runners.eval_runners,4 e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.runners.sqlgen_runners,20 -e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.scorers.llmrater.model_config,autoctx/experiments/my-alloydb-tuning-experiment/eval_configs/llmrater_config.yaml -e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.reporting.csv.output_directory,autoctx/experiments/my-alloydb-tuning-experiment/eval_reports/ +e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.scorers.llmrater.model_config,autoctx/experiments/crema-eval-experiment/eval_configs/llmrater_config.yaml +e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.reporting.csv.output_directory,autoctx/experiments/crema-eval-experiment/eval_reports/ e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,experiment_config.orchestrator,oneshot e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,model_config.generator,query_data_api e637c96e-ab21-411b-99a2-47ec808ad968,2026-05-07 19:52:28.203633,model_config.project_id,cloud-db-nl2sql diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/evals.csv diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/scores.csv diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv similarity index 100% rename from evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv rename to evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/crema-eval-experiment/v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/summary.csv diff --git a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md b/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md deleted file mode 100644 index 57443594..00000000 --- a/evals/core-cujs/workspace_post_evaluation/.context-engineering/experiments/my-alloydb-tuning-experiment/state.md +++ /dev/null @@ -1,16 +0,0 @@ -# Hill-Climbing Experiment: my-alloydb-tuning-experiment - -## Metadata -- Workspace: ./.context-engineering/experiments/my-alloydb-tuning-experiment/ -- Golden dataset: ./golden.json -- DB source: my-alloydb -- Context Store: projects/astana-transformation/locations/us-central1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx - -## Iteration Log - -### v0 (base) -- Context resource: projects/astana-transformation/locations/us-central1/contextSetGroups/my-alloydb-tuning-experiment/contextSets/autoctx@v0 -- Score: 0.75 -- Analysis: (baseline — no analysis) -- Eval report: v0/eval/eval_reports/e637c96e-ab21-411b-99a2-47ec808ad968/ -- Notes: baseline uploaded from bootstrap_context.json diff --git a/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml index c66f2491..2624bc1e 100644 --- a/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml +++ b/evals/core-cujs/workspace_post_expansion/.context-engineering/tools.yaml @@ -1,10 +1,10 @@ kind: source name: my-alloydb type: alloydb-postgres -project: astana-transformation -region: us-east4 -cluster: juexinw-test -instance: juexinw-test-primary +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary database: financial --- kind: tool diff --git a/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml b/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml index c66f2491..2624bc1e 100644 --- a/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml +++ b/evals/core-cujs/workspace_post_init/.context-engineering/tools.yaml @@ -1,10 +1,10 @@ kind: source name: my-alloydb type: alloydb-postgres -project: astana-transformation -region: us-east4 -cluster: juexinw-test -instance: juexinw-test-primary +project: cloud-db-nl2sql +region: us-central1 +cluster: whaoyu-test +instance: whaoyu-test-primary database: financial --- kind: tool From b75522de9d282cadd790e7e05298a19e0bb2cb34 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Tue, 21 Jul 2026 23:33:57 -0700 Subject: [PATCH 12/19] chore(core-cujs): bake context and experiment name into hillclimb-fresh starting prompt --- evals/core-cujs/dataset.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/core-cujs/dataset.json b/evals/core-cujs/dataset.json index 131c886f..66c892ca 100644 --- a/evals/core-cujs/dataset.json +++ b/evals/core-cujs/dataset.json @@ -56,7 +56,7 @@ }, { "id": "core-cujs:hillclimb-fresh", - "starting_prompt": "improve my context set from scratch using the golden dataset. If you need any details, stop and ask before proceeding.", + "starting_prompt": "improve my context set from scratch using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. Use experiment name 'crema-eval-experiment' and project 'cloud-db-nl2sql'.", "conversation_plan": "Ask the agent to run hillclimb using the local base context at ./bootstrap_context.json and the golden dataset at ./golden.json. If prompted for experiment name, use 'crema-eval-experiment'. Terminate immediately after the agent successfully performs the hillclimbing task, even if the agent prompts you to continue.", "expected_trajectory": [], "kind": "agents", From 3e8bee78d7c1d9cc1e011bde2a44bfee21a6fc24 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Tue, 21 Jul 2026 23:35:43 -0700 Subject: [PATCH 13/19] chore(evals): lower gemini judge execs_per_minute to 20 --- evals/model_configs/gemini_model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/model_configs/gemini_model.yaml b/evals/model_configs/gemini_model.yaml index 9488e69c..560a5b81 100644 --- a/evals/model_configs/gemini_model.yaml +++ b/evals/model_configs/gemini_model.yaml @@ -1,4 +1,4 @@ generator: gcp_vertex_gemini vertex_model: gemini-3-flash-preview base_prompt: "" -execs_per_minute: 70 +execs_per_minute: 20 From 78213d2839bfad135b0df7169832cffb91e386b7 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Wed, 22 Jul 2026 14:27:34 -0700 Subject: [PATCH 14/19] fix(client): switch Context Store + QueryData to prod; fall back to default project for quota --- .../common/context_store_client.py | 24 +++++++++++-------- .../evaluate/db_generators/base.py | 6 ----- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/google/cloud/db_context_enrichment/common/context_store_client.py b/src/google/cloud/db_context_enrichment/common/context_store_client.py index 07e8265b..5a931029 100644 --- a/src/google/cloud/db_context_enrichment/common/context_store_client.py +++ b/src/google/cloud/db_context_enrichment/common/context_store_client.py @@ -15,7 +15,7 @@ from google.cloud.db_context_enrichment.model import context -CONTEXT_STORE_ENDPOINT = "https://autopush-dataplex.sandbox.googleapis.com" +CONTEXT_STORE_ENDPOINT = "https://dataplex.googleapis.com" CONTEXT_STORE_LOCATION = "us-central1" API_VERSION = "v1" DEFAULT_OAUTH_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",) @@ -36,25 +36,29 @@ class ContextStoreClient: path from IDs (`ensure_*`) take `project_id` as an explicit argument; operations that take a fully-qualified resource name don't need it. - The quota project (billing / quota) is always taken from ADC's - `quota_project_id` and sent as `X-Goog-User-Project` on every request. + The quota project (billing / quota) is taken from ADC's + `quota_project_id` if set, else the project ADC returns alongside the + credentials (user default via `gcloud config get-value project`, or the + SA key's `project_id`). Sent as `X-Goog-User-Project` on every request. + Omitted if neither is available. """ def __init__(self): try: - credentials, _ = google.auth.default(scopes=DEFAULT_OAUTH_SCOPES) + credentials, default_project = google.auth.default( + scopes=DEFAULT_OAUTH_SCOPES + ) except google.auth.exceptions.DefaultCredentialsError as e: raise RuntimeError( "No Application Default Credentials found. Run " "'gcloud auth application-default login' first." ) from e - if not credentials.quota_project_id: - raise RuntimeError( - "No quota project set. Run " - "'gcloud auth application-default set-quota-project '." - ) + # Matches how the generated Google SDKs pick a billing project: + # explicit ADC quota project first, then ADC's default project. + quota_project = credentials.quota_project_id or default_project self._session = auth_requests.AuthorizedSession(credentials) - self._session.headers["X-Goog-User-Project"] = credentials.quota_project_id + if quota_project: + self._session.headers["X-Goog-User-Project"] = quota_project def ensure_context_set_group(self, project_id: str, csg_id: str) -> str: """Return a CSG's full resource name, creating it if absent. diff --git a/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py b/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py index debecac1..d393f7d4 100644 --- a/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py +++ b/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py @@ -5,11 +5,6 @@ import yaml from google.protobuf.json_format import MessageToDict -# Pinned to the QueryData autopush sandbox to match the Context Store endpoint -# used by `context_store_client`. When both APIs land on production, drop this -# and let evalbench default to the production QueryData endpoint. -_QUERY_DATA_API_ENDPOINT = "autopush-geminidataanalytics.sandbox.googleapis.com" - class BaseDBConfigGenerator(ABC): """ @@ -73,7 +68,6 @@ def generate_model_config(self, context_set_id: str) -> str: "generator": "query_data_api", "project_id": self.params.get("project"), "location": self.params.get("region") or "global", - "api_endpoint": _QUERY_DATA_API_ENDPOINT, "context": query_context_dict, } From 842f5f783fb2582e15e0d05903346a9d3f96f2ca Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Wed, 22 Jul 2026 14:33:24 -0700 Subject: [PATCH 15/19] chore(dev-plugin): use placeholder instead of hardcoded local path --- dev-plugin/gemini-extension.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-plugin/gemini-extension.json b/dev-plugin/gemini-extension.json index b8662943..5d1f2ead 100644 --- a/dev-plugin/gemini-extension.json +++ b/dev-plugin/gemini-extension.json @@ -7,7 +7,7 @@ "args": [ "run", "--project", - "/Users/juexinw/Workspace/nla/db-context-enrichment", + "", "google-cloud-db-context-engineering" ] }, From 433b2515623a89366fb5d92e9f83249fb2c7688a Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Wed, 22 Jul 2026 15:24:35 -0700 Subject: [PATCH 16/19] chore(ci,core-cujs): add 45-min step timeouts; bake required coordinates/filenames into starting prompts --- .ci/cloudbuild.yaml | 12 ++++++++++++ evals/core-cujs/dataset.json | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.ci/cloudbuild.yaml b/.ci/cloudbuild.yaml index 97b3dceb..146353ea 100644 --- a/.ci/cloudbuild.yaml +++ b/.ci/cloudbuild.yaml @@ -83,6 +83,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -94,6 +95,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -107,6 +109,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -118,6 +121,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -129,6 +133,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -140,6 +145,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -152,6 +158,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -164,6 +171,7 @@ steps: entrypoint: 'bash' secretEnv: ['ADC_KEY'] waitFor: ['build-extension', 'preflight'] + timeout: 2700s env: - 'EVAL_GCP_PROJECT_ID=$PROJECT_ID' - 'EVAL_GCP_PROJECT_REGION=$_PROJECT_LOCATION' @@ -280,3 +288,7 @@ substitutions: options: defaultLogsBucketBehavior: REGIONAL_USER_OWNED_BUCKET + +# Steps run in parallel; overall timeout must cover the slowest step +# (45 min) plus build-extension + preflight + upload + gate overhead. +timeout: 3600s diff --git a/evals/core-cujs/dataset.json b/evals/core-cujs/dataset.json index 66c892ca..e046b06e 100644 --- a/evals/core-cujs/dataset.json +++ b/evals/core-cujs/dataset.json @@ -20,7 +20,7 @@ }, { "id": "core-cujs:dataset-expansion", - "starting_prompt": "Use context engineering, generate an evaluation dataset consisting of 5 questions by expanding my evaluation dataset golden.json with more diverse examples using variation strategies.", + "starting_prompt": "Use context engineering, generate an evaluation dataset consisting of 5 questions by expanding my evaluation dataset golden.json with more diverse examples using variation strategies. Name the audit report files 'evalset_expansion_report_pair_level.md' and 'evalset_expansion_report_dataset_level.md'.", "conversation_plan": "Ask the agent to expand the existing golden.json evaluation dataset by applying variation strategies to the existing pairs. The agent should apply all six strategies: paraphrasing, merging, difficulty adjustment (both upscaling and simplification), distraction injection, linguistic variation (typos and synonyms), and value substitution (which requires running SELECT DISTINCT queries on the database). When the agent presents an expansion plan, confirm it and approve all strategies. When the agent runs SQL execution validation or asks for approval of the validation reports, approve them. You should terminate the conversation immediately after the agent successfully writes the expanded pairs to golden.json, even if the agent prompts you to perform evaluation or take next steps. At the end of the conversation, verify that the agent produced the following output files: golden.json, evalset_expansion_report_pair_level.md, evalset_expansion_report_dataset_level.md.", "expected_trajectory": [], "kind": "agents", @@ -38,7 +38,7 @@ }, { "id": "core-cujs:evaluate", - "starting_prompt": "evaluate my context set against the golden dataset. If you need any details, stop and ask before proceeding.", + "starting_prompt": "evaluate the golden dataset at ./golden.json against the local ContextSet at ./bootstrap_context.json. Use project 'cloud-db-nl2sql', experiment 'crema-eval-experiment', cs_id 'autoctx', version 'v0'.", "conversation_plan": "Ask the agent to evaluate the golden dataset at ./golden.json against the local ContextSet at ./bootstrap_context.json. If prompted for Context Store coordinates, use project_id=cloud-db-nl2sql, csg_id=crema-eval-experiment, cs_id=autoctx, version=v0. Terminate immediately after the agent presents evaluation results, even if it prompts to continue.", "expected_trajectory": [], "kind": "agents", @@ -47,7 +47,7 @@ }, { "id": "core-cujs:hillclimb-continue", - "starting_prompt": "improve the context set based on eval failure. If you need any details, stop and ask before proceeding.", + "starting_prompt": "improve the context set based on eval failure. Resume the in-progress experiment 'crema-eval-experiment' at .context-engineering/experiments/crema-eval-experiment/ (already has a v0 baseline). Use project 'cloud-db-nl2sql'.", "conversation_plan": "Ask the agent to perform the hillclimbing to improve the context set based on eval failure. The workspace already contains an in-progress experiment at .context-engineering/experiments/crema-eval-experiment/ with a v0 baseline. If prompted for experiment name, use 'crema-eval-experiment'. You should terminate the conversation immediately after the agent successfully performed the hillclimbing task, even if the agent prompts you to perform evaluation or take next steps.", "expected_trajectory": [], "kind": "agents", From 6545c4e2ccd32d0da19048a913256bfa8fc49a64 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Fri, 24 Jul 2026 13:49:59 -0700 Subject: [PATCH 17/19] fix: align tests/docstrings with prod Context Store endpoint and harden release.yml (pin actions, fix template-injection) --- .github/workflows/release.yml | 23 ++++++++------ .../common/context_store_client.py | 4 +-- .../common/context_store_client_test.py | 31 ++++++++++++++++--- .../evaluate/db_generators/alloydb_test.py | 1 - .../evaluate/db_generators/mysql_test.py | 1 - .../evaluate/db_generators/postgres_test.py | 1 - .../evaluate/db_generators/spanner_test.py | 1 - .../evaluate/evaluate_generator_test.py | 1 - 8 files changed, 42 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 571ecc69..52dce5cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,11 +40,11 @@ jobs: steps: # Checkout the repository code - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 # Install the `uv` Python package manager which handles fast installations - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: enable-cache: true cache-dependency-glob: "uv.lock" @@ -77,7 +77,7 @@ jobs: - name: Download mcp-toolbox binary shell: bash run: | - DOWNLOAD_URL="https://storage.googleapis.com/mcp-toolbox-for-databases/v${{ env.TOOLBOX_VERSION }}/${{ matrix.toolbox_os }}/${{ matrix.toolbox_arch }}/toolbox${{ matrix.toolbox_ext }}" + DOWNLOAD_URL="https://storage.googleapis.com/mcp-toolbox-for-databases/v${TOOLBOX_VERSION}/${{ matrix.toolbox_os }}/${{ matrix.toolbox_arch }}/toolbox${{ matrix.toolbox_ext }}" echo "Downloading toolbox from: ${DOWNLOAD_URL}" curl -L --fail -o "toolbox${{ matrix.toolbox_ext }}" "${DOWNLOAD_URL}" chmod +x "toolbox${{ matrix.toolbox_ext }}" @@ -91,7 +91,7 @@ jobs: else ARCHIVE="${{ matrix.platform_name }}.${{ matrix.arch_name }}.evalbench.tar.gz" fi - DOWNLOAD_URL="https://github.com/GoogleCloudPlatform/evalbench/releases/download/v${{ env.EVALBENCH_VERSION }}/${ARCHIVE}" + DOWNLOAD_URL="https://github.com/GoogleCloudPlatform/evalbench/releases/download/v${EVALBENCH_VERSION}/${ARCHIVE}" echo "Downloading evalbench from: ${DOWNLOAD_URL}" curl -L --fail -o "${ARCHIVE}" "${DOWNLOAD_URL}" @@ -162,7 +162,7 @@ jobs: # Upload the archives to GitHub Actions artifacts so the publish job can access them - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: binary-${{ matrix.platform_name }}-${{ matrix.arch_name }} path: | @@ -180,7 +180,7 @@ jobs: steps: # Download all platform archives generated by the build-artifacts job - name: Download all build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: release-assets pattern: binary-* @@ -189,7 +189,7 @@ jobs: # Attach the downloaded archives as assets to the created GitHub Release - name: Upload Release Assets if: github.event_name == 'release' - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2 with: tag_name: ${{ github.event.release.tag_name }} files: release-assets/* @@ -204,13 +204,15 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 # Strip leading 'v' from the release tag (e.g. v0.5.1 -> 0.5.1) since PyPI versions are unprefixed. # Poll PyPI for up to 6 hours (72 attempts x 5min) waiting for the wheel to be indexed. - name: Wait for PyPI release + env: + TAG_NAME: ${{ github.event.release.tag_name }} run: | - TAG="${{ github.event.release.tag_name }}" + TAG="${TAG_NAME}" VERSION="${TAG#v}" PKG="google-cloud-db-context-engineering" for i in {1..72}; do @@ -229,8 +231,9 @@ jobs: - name: Open PR to pin marketplace ref env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ github.event.release.tag_name }} run: | - TAG="${{ github.event.release.tag_name }}" + TAG="${TAG_NAME}" BRANCH="chore/pin-marketplace-${TAG}" jq --arg ref "$TAG" '.plugins[0].source.ref = $ref' \ .claude-plugin/marketplace.json > tmp && mv tmp .claude-plugin/marketplace.json diff --git a/src/google/cloud/db_context_enrichment/common/context_store_client.py b/src/google/cloud/db_context_enrichment/common/context_store_client.py index 5a931029..d2ff0bfc 100644 --- a/src/google/cloud/db_context_enrichment/common/context_store_client.py +++ b/src/google/cloud/db_context_enrichment/common/context_store_client.py @@ -1,7 +1,7 @@ """REST client for the Context Store API. Hand-rolled because the service is GOOGLE_INTERNAL and has no public SDK. -All calls target the autopush sandbox endpoint pinned in this module. +All calls target the prod Dataplex endpoint pinned in this module. """ import json @@ -30,7 +30,7 @@ class ContextStoreClient: - """REST client for the Context Store API (autopush sandbox). + """REST client for the Context Store API (prod Dataplex). The client is stateless w.r.t. project. Operations that build a resource path from IDs (`ensure_*`) take `project_id` as an explicit argument; diff --git a/tests/google/cloud/db_context_enrichment/common/context_store_client_test.py b/tests/google/cloud/db_context_enrichment/common/context_store_client_test.py index d2ee4960..ab6d2f36 100644 --- a/tests/google/cloud/db_context_enrichment/common/context_store_client_test.py +++ b/tests/google/cloud/db_context_enrichment/common/context_store_client_test.py @@ -81,21 +81,44 @@ def raise_default(scopes=None): ContextStoreClient() -def test_constructor_raises_when_no_quota_project(monkeypatch): +def test_constructor_omits_user_project_header_when_none_available(monkeypatch): fake_credentials = MagicMock() fake_credentials.quota_project_id = None monkeypatch.setattr( "google.auth.default", lambda scopes=None: (fake_credentials, None), ) + fake_session = MagicMock() + fake_session.headers = {} monkeypatch.setattr( context_store_client.auth_requests, "AuthorizedSession", - lambda creds: MagicMock(), + lambda creds: fake_session, ) - with pytest.raises(RuntimeError, match="No quota project"): - ContextStoreClient() + client = ContextStoreClient() + + assert "X-Goog-User-Project" not in client._session.headers + + +def test_constructor_falls_back_to_default_project_for_quota(monkeypatch): + fake_credentials = MagicMock() + fake_credentials.quota_project_id = None + monkeypatch.setattr( + "google.auth.default", + lambda scopes=None: (fake_credentials, "adc-default-project"), + ) + fake_session = MagicMock() + fake_session.headers = {} + monkeypatch.setattr( + context_store_client.auth_requests, + "AuthorizedSession", + lambda creds: fake_session, + ) + + client = ContextStoreClient() + + assert client._session.headers["X-Goog-User-Project"] == "adc-default-project" def test_constructor_sends_quota_project_header(monkeypatch): diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py index 1eab9060..036c35c8 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/alloydb_test.py @@ -49,7 +49,6 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "us-west1", - "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "alloydb": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py index 36d0e29c..83767bfa 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/mysql_test.py @@ -47,7 +47,6 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "us-west1", - "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "cloud_sql_reference": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py index d8f1af57..93627f4b 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/postgres_test.py @@ -47,7 +47,6 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "us-west1", - "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "cloud_sql_reference": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py index 2b9866b9..1814cedf 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/db_generators/spanner_test.py @@ -44,7 +44,6 @@ def test_generate_model_config(mock_params): "generator": "query_data_api", "project_id": "test-project", "location": "global", - "api_endpoint": "autopush-geminidataanalytics.sandbox.googleapis.com", "context": { "datasource_references": { "spanner_reference": { diff --git a/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py b/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py index 8454da5e..ff746932 100644 --- a/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py +++ b/tests/google/cloud/db_context_enrichment/evaluate/evaluate_generator_test.py @@ -152,7 +152,6 @@ def test_generate_evalbench_configs(): generator: query_data_api project_id: test-project location: us-central1 - api_endpoint: autopush-geminidataanalytics.sandbox.googleapis.com context: datasource_references: cloud_sql_reference: From 32a6f0bb90cc8a2ce4a3407d9df96f8886a66d6a Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Tue, 28 Jul 2026 16:47:35 -0700 Subject: [PATCH 18/19] fix(release): stop referencing removed GEMINI.md in Prepare distribution and manifests --- .github/workflows/release.yml | 3 +-- gemini-extension.json | 1 - plugin/gemini-extension.json | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52dce5cb..f50f2583 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -142,8 +142,7 @@ jobs: TOOLBOX_NAME="\${extensionPath}/skills/autoctx-init/scripts/toolbox" fi - jq ".contextFileName = \"GEMINI.md\" | .mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \".context-engineering/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json - cp plugin/GEMINI.md staging/ + jq ".mcpServers.mcp_db_context_engineering.command = \"$BINARY_NAME\" | .mcpServers.mcp_db_context_engineering.args = [] | .mcpServers.mcp_toolbox = {\"command\": \"$TOOLBOX_NAME\", \"args\": [\"--stdio\", \"--config\", \".context-engineering/tools.yaml\"]}" plugin/gemini-extension.json > staging/gemini-extension.json cp LICENSE staging/ # Create a compressed tarball archive for Linux and macOS distributions diff --git a/gemini-extension.json b/gemini-extension.json index 4bf11761..c6eb6c3b 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,7 +1,6 @@ { "name": "google-cloud-db-context-engineering", "version": "0.7.0", - "contextFileName": "./plugin/GEMINI.md", "mcpServers": { "db-context-engineering": { "command": "uvx", diff --git a/plugin/gemini-extension.json b/plugin/gemini-extension.json index 1885bc26..9711e683 100644 --- a/plugin/gemini-extension.json +++ b/plugin/gemini-extension.json @@ -1,7 +1,6 @@ { "name": "google-cloud-db-context-engineering", "version": "0.7.0", - "contextFileName": "./GEMINI.md", "mcpServers": { "mcp_db_context_engineering": { "command": "uv", From 30a5d3dd19d23695e94e733005943e3eae316a72 Mon Sep 17 00:00:00 2001 From: Juexin Wang Date: Tue, 28 Jul 2026 16:47:35 -0700 Subject: [PATCH 19/19] chore: sync uv.lock project version to 0.7.0 --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 8ab3272a..948fce8e 100644 --- a/uv.lock +++ b/uv.lock @@ -724,7 +724,7 @@ wheels = [ [[package]] name = "google-cloud-db-context-engineering" -version = "0.6.0" +version = "0.7.0" source = { editable = "." } dependencies = [ { name = "fastmcp" },