From dc072809a4953652a8ace9bdfa0052296cb707d1 Mon Sep 17 00:00:00 2001 From: g-lynnzee <139825992+g-lynnzee@users.noreply.github.com> Date: Sun, 21 Jun 2026 21:43:10 -0700 Subject: [PATCH 1/3] fix: improvements to ensure agent doesn't get stuck on local CUJ run tested: fixes used to run local CUJs for me. --- evals/.gitignore | 3 +++ evals/README.md | 25 +++++++++++++++++++------ evals/model_configs/gemini_model.yaml | 2 ++ 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 evals/.gitignore diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 00000000..523f67aa --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,3 @@ +results/ +.venv/ +uv.toml diff --git a/evals/README.md b/evals/README.md index 878f40bb..9329cade 100644 --- a/evals/README.md +++ b/evals/README.md @@ -4,7 +4,7 @@ This directory contains the evaluation suite for the main functionalities of the ## Overview -The evaluation uses the [evalbench](https://github.com/GoogleCloudPlatform/evalbench) framework with the Gemini CLI and Claude Code orchestrator to run a set of simulated user tasks against the agent. +The evaluation uses the [evalbench](https://github.com/GoogleCloudPlatform/evalbench) framework with the various agent harness orchestrators to run a set of simulated user tasks. ## Configuration Files @@ -33,12 +33,25 @@ The suite evaluates the agent across several dimensions using the following scor ## How to Run -From the `evals/` directory, execute the evaluation using latest [evalbench](https://github.com/GoogleCloudPlatform/evalbench/releases): - +From the `evals/` directory, run the evaluation using `uvx`: ```bash cd evals/ -evalbench run core-cujs/run_gemini_cli.yaml # Gemini CLI -evalbench run core-cujs/run_claude.yaml # Claude Code +UV_CONFIG_FILE=uv.toml uvx --index-url https://pypi.org/simple/ google-evalbench --experiment_config=core-cujs/run_gemini_cli.yaml ``` -Results will be generated in the `results/` directory as CSV reports. +### Local Execution Caveats +Unlike CI (which auto-injects values and queries the metadata server), running locally requires these specific alignments: + +1. **Node Version**: Switch your active shell session to Node v20+ before running to support modern regular expressions: + ```bash + source ~/.nvm/nvm.sh && nvm use 20 + ``` +2. **Dirty State Cleanup**: If a run crashes midway, wipe the dirty extension installation before retrying: + ```bash + rm -rf evals/.venv/fake_home/.gemini/extensions/google-cloud-db-context-engineering + ``` +3. **Environment Variables**: Before executing a run, ensure you have exported your GCP project ID and the global endpoint location: + ```bash + export GOOGLE_CLOUD_PROJECT="your-gcp-project-id" + export GOOGLE_CLOUD_LOCATION="global" + ``` diff --git a/evals/model_configs/gemini_model.yaml b/evals/model_configs/gemini_model.yaml index 4bb440a0..37cff32a 100644 --- a/evals/model_configs/gemini_model.yaml +++ b/evals/model_configs/gemini_model.yaml @@ -1,4 +1,6 @@ generator: gcp_vertex_gemini vertex_model: gemini-3-flash-preview +gcp_project_id: "${GOOGLE_CLOUD_PROJECT}" +gcp_region: "${GOOGLE_CLOUD_LOCATION:global}" base_prompt: "" execs_per_minute: 5 From 611a2c2a2d453d1d1678d6503f0390481822d5e6 Mon Sep 17 00:00:00 2001 From: g-lynnzee <139825992+g-lynnzee@users.noreply.github.com> Date: Sat, 18 Jul 2026 22:34:42 -0700 Subject: [PATCH 2/3] feat(evaluate): add automatic REST fallback for unreleased proto fields Summary Implements an automated transport fallback mechanism for how Evalbench calls QueryData to seamlessly support unreleased/pre-release protocol buffer fields (such as ) without requiring end users to modify tools.yaml. The goal is to enable developing and releasing pre-public features in context engineering agent, such as to private preview customers. --- plugin/skills/autoctx-evaluate/SKILL.md | 4 +- pyproject.toml | 1 + .../evaluate/db_generators/base.py | 22 +++- .../evaluate/evaluate_generator.py | 119 ++++++++++++++++++ 4 files changed, 139 insertions(+), 7 deletions(-) diff --git a/plugin/skills/autoctx-evaluate/SKILL.md b/plugin/skills/autoctx-evaluate/SKILL.md index 719d1378..b2ef1eac 100644 --- a/plugin/skills/autoctx-evaluate/SKILL.md +++ b/plugin/skills/autoctx-evaluate/SKILL.md @@ -68,8 +68,8 @@ Follow these steps exactly in order: - You do not need to manually write or extract file contents. Verify that the files have materialized if needed. 4. **Evalbench Run Integration:** - - Trigger the `run_shell_command` natively to execute the evaluation from the ROOT of the workspace using the following exact command template: - `uvx google-evalbench@1.9.0 --experiment_config=autoctx/experiments//eval_configs/run_config.yaml` + - Trigger the `run_shell_command` natively to execute the evaluation runner with automatic 3-tiered REST API fallback: + `uv run autoctx-eval ` - Check the command outputs to ensure the evaluation reports materialize in the respective `autoctx/experiments//eval_reports/` directory. ## Output diff --git a/pyproject.toml b/pyproject.toml index 1c38b225..784d86e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ [project.scripts] google-cloud-db-context-engineering = "google.cloud.db_context_enrichment.main:mcp.run" +autoctx-eval = "google.cloud.db_context_enrichment.evaluate.evaluate_generator:cli_main" [project.optional-dependencies] test = [ diff --git a/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py b/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py index d393f7d4..24b3b3e9 100644 --- a/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py +++ b/src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py @@ -58,11 +58,17 @@ def generate_model_config(self, context_set_id: str) -> str: """ datasource_ref = self.build_datasource_reference(context_set_id) - query_context = gda.QueryDataContext(datasource_references=datasource_ref) - - query_context_dict = MessageToDict( - query_context._pb, preserving_proto_field_name=True - ) + if isinstance(datasource_ref, dict): + query_context_dict = { + "datasource_references": datasource_ref + } + else: + query_context = gda.QueryDataContext( + datasource_references=datasource_ref + ) + query_context_dict = MessageToDict( + query_context._pb, preserving_proto_field_name=True + ) model_config = { "generator": "query_data_api", @@ -71,6 +77,12 @@ def generate_model_config(self, context_set_id: str) -> str: "context": query_context_dict, } + if self.params.get("use_rest_api"): + model_config["use_rest_api"] = True + + if self.params.get("api_endpoint"): + model_config["api_endpoint"] = self.params.get("api_endpoint") + return yaml.safe_dump( model_config, sort_keys=False, default_flow_style=False ).strip() diff --git a/src/google/cloud/db_context_enrichment/evaluate/evaluate_generator.py b/src/google/cloud/db_context_enrichment/evaluate/evaluate_generator.py index 2945b197..4efbf3be 100644 --- a/src/google/cloud/db_context_enrichment/evaluate/evaluate_generator.py +++ b/src/google/cloud/db_context_enrichment/evaluate/evaluate_generator.py @@ -1,6 +1,8 @@ import json +import logging import os import re +import subprocess import textwrap from typing import Any @@ -237,3 +239,120 @@ def _convert_dataset(dataset_path: str, dialect: str) -> str: return json.dumps(converted, indent=2) except Exception as e: raise ValueError(f"Failed to convert dataset at {dataset_path}: {e}") + + +_STAGING_API_ENDPOINT = "staging-geminidataanalytics.sandbox.googleapis.com" + + +_PROTO_FIELD_ERROR_PATTERNS = ( + "attributeerror", + "typeerror", + "valueerror", + "protocol message", + "unknown field", + "invalid field", + "has no attribute", + "querydatacontext", + "datasourcereferences", +) + + +def _exec_evalbench(cmd: list[str]) -> tuple[int, str]: + """Executes EvalBench command and returns (returncode, combined_output).""" + res = subprocess.run(cmd, capture_output=True, text=True) + out = (res.stderr or "") + (res.stdout or "") + return res.returncode, out + + +def _is_proto_field_error(output: str) -> bool: + """Returns True if the output contains errors related to missing/unreleased proto fields.""" + out_lower = output.lower() + return any(pattern in out_lower for pattern in _PROTO_FIELD_ERROR_PATTERNS) + + +def _update_model_config( + model_config_path: str, + use_rest_api: bool = True, + api_endpoint: str | None = None, +) -> None: + """Updates model_config.yaml with REST API transport flags.""" + with open(model_config_path) as f: + cfg = yaml.safe_load(f) or {} + + cfg["use_rest_api"] = use_rest_api + if api_endpoint: + cfg["api_endpoint"] = api_endpoint + + with open(model_config_path, "w") as f: + yaml.safe_dump(cfg, f, sort_keys=False, default_flow_style=False) + + +def run_evaluation(experiment_name: str) -> None: + """ + Executes EvalBench evaluation for an experiment, using standard SDK gRPC client + and falling back to REST (supports nonpublic fields). + """ + logger = logging.getLogger(__name__) + eval_configs_dir = f"autoctx/experiments/{experiment_name}/eval_configs" + run_config_path = os.path.join(eval_configs_dir, RUN_CONFIG_NAME) + model_config_path = os.path.join(eval_configs_dir, MODEL_CONFIG_NAME) + + cmd = [ + "uvx", + "google-evalbench@1.9.0", + f"--experiment_config={run_config_path}", + ] + + # 1. Standard SDK gRPC Execution + logger.info( + f"Running EvalBench evaluation for experiment: {experiment_name}" + ) + code, output = _exec_evalbench(cmd) + if code == 0 and not _is_proto_field_error(output): + logger.info("EvalBench completed successfully via gRPC SDK.") + return + + if not _is_proto_field_error(output): + logger.error( + f"EvalBench execution failed with non-proto error:\n{output[:500]}" + ) + raise RuntimeError( + f"EvalBench execution failed with exit code {code}:\n{output}" + ) + + # 2. REST API Fallback Tiers (Production REST, then Staging REST) + rest_tiers = [ + ("Production REST API", True, None), + (f"Staging REST API ({_STAGING_API_ENDPOINT})", True, _STAGING_API_ENDPOINT), + ] + + for tier_name, use_rest, endpoint in rest_tiers: + logger.info(f"Attempting evaluation fallback via {tier_name}...") + try: + _update_model_config( + model_config_path, use_rest_api=use_rest, api_endpoint=endpoint + ) + code, output = _exec_evalbench(cmd) + if code == 0: + logger.info(f"Evaluation completed successfully via {tier_name}.") + return + except Exception as err: + logger.debug(f"{tier_name} execution failed: {err}") + + logger.error(f"Evaluation failed across all transport modes:\n{output[:500]}") + raise RuntimeError( + f"EvalBench evaluation failed for experiment '{experiment_name}'.\n" + "You may be attempting to use an unreleased or non-public QueryData feature. Please reach out to your accounts team on for access." + ) + + +def cli_main() -> None: + """CLI entrypoint for autoctx-eval command.""" + import sys + + if len(sys.argv) < 2: + print("Usage: autoctx-eval ") + sys.exit(1) + + experiment_name = sys.argv[1] + run_evaluation(experiment_name) From c5f830ae37ab558f0d08f8c7ffabea7156f78ec8 Mon Sep 17 00:00:00 2001 From: g-lynnzee <139825992+g-lynnzee@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:19:50 -0700 Subject: [PATCH 3/3] ci(cloudbuild): skip cloudbuild for documentation-only PRs Don't waste resources here. --- .ci/cloudbuild.yaml | 10 ++++++++++ .ci/run_claude_code.sh | 6 ++++++ .ci/run_gemini_cli.sh | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/.ci/cloudbuild.yaml b/.ci/cloudbuild.yaml index acf17d52..a4f338d8 100644 --- a/.ci/cloudbuild.yaml +++ b/.ci/cloudbuild.yaml @@ -76,6 +76,16 @@ steps: echo "PR does not have 'ci:eval-claude' label. Skipping Claude Code evals." fi + echo "Checking changed files for documentation-only PR..." + curl -s -o pr_files.json -H "Authorization: token $$GITHUB_TOKEN" \ + "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER/files" + + NON_DOC_COUNT=$(jq '[.[]? | select(.filename | test("^(README\\.md|CHANGELOG\\.md|CONTRIBUTING\\.md|LICENSE|\\.gitignore|docs/)") | not)] | length' pr_files.json 2>/dev/null || echo 1) + if [ "$$NON_DOC_COUNT" -eq 0 ]; then + echo "Documentation-only PR detected (only README.md, docs/, etc. modified)." + touch /workspace/IS_DOCS_ONLY + fi + # --- Smoke tests (always run, regardless of PR labels) --- # Cheap signal that the agent + plugin + MCP wiring loads at all. - id: eval-gemini-cli-smoke-test diff --git a/.ci/run_claude_code.sh b/.ci/run_claude_code.sh index 20a76c1a..9f0caf07 100755 --- a/.ci/run_claude_code.sh +++ b/.ci/run_claude_code.sh @@ -11,6 +11,12 @@ set -e SUITE="${1:?usage: run_claude_code.sh }" SUT="claude-code" +# Documentation-only PRs skip smoke tests unless explicitly requested via ci:eval-claude label. +if [ -f /workspace/IS_DOCS_ONLY ] && [ ! -f /workspace/SHOULD_RUN_CLAUDE_CODE ]; then + echo "Documentation-only PR without ci:eval-claude label; skipping ${SUT}/${SUITE}." + exit 0 +fi + # The smoke-test suite always runs regardless of PR labels; other suites are # gated by the ci:eval-claude label (marker written by the preflight step). if [ "${SUITE}" != "smoke-test" ] && [ ! -f /workspace/SHOULD_RUN_CLAUDE_CODE ]; then diff --git a/.ci/run_gemini_cli.sh b/.ci/run_gemini_cli.sh index 7f2a58e0..4fc2d1b8 100755 --- a/.ci/run_gemini_cli.sh +++ b/.ci/run_gemini_cli.sh @@ -11,6 +11,12 @@ set -e SUITE="${1:?usage: run_gemini_cli.sh }" SUT="gemini-cli" +# Documentation-only PRs skip smoke tests unless explicitly requested via ci:eval label. +if [ -f /workspace/IS_DOCS_ONLY ] && [ ! -f /workspace/SHOULD_RUN_GEMINI_CLI ]; then + echo "Documentation-only PR without ci:eval label; skipping ${SUT}/${SUITE}." + exit 0 +fi + # The smoke-test suite always runs regardless of PR labels; other suites are # gated by the ci:eval label (marker written by the preflight step). if [ "${SUITE}" != "smoke-test" ] && [ ! -f /workspace/SHOULD_RUN_GEMINI_CLI ]; then