ci(cloudbuild): skip cloudbuild for documentation-only PRsIgnore docs cloudrun

.ci/cloudbuild.yaml

@@ -76,6 +76,16 @@ steps:
           echo "PR does not have 'ci:eval-claude' label. Skipping Claude Code evals."
         fi
 
+        echo "Checking changed files for documentation-only PR..."
+        curl -s -o pr_files.json -H "Authorization: token $$GITHUB_TOKEN" \
+          "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER/files"
+
+        NON_DOC_COUNT=$(jq '[.[]? | select(.filename | test("^(README\\.md|CHANGELOG\\.md|CONTRIBUTING\\.md|LICENSE|\\.gitignore|docs/)") | not)] | length' pr_files.json 2>/dev/null || echo 1)
+        if [ "$$NON_DOC_COUNT" -eq 0 ]; then
+          echo "Documentation-only PR detected (only README.md, docs/, etc. modified)."
+          touch /workspace/IS_DOCS_ONLY
+        fi
+
   # --- Smoke tests (always run, regardless of PR labels) ---
   # Cheap signal that the agent + plugin + MCP wiring loads at all.
   - id: eval-gemini-cli-smoke-test

.ci/run_claude_code.sh

@@ -11,6 +11,12 @@ set -e
 SUITE="${1:?usage: run_claude_code.sh <suite-name>}"
 SUT="claude-code"
 
+# Documentation-only PRs skip smoke tests unless explicitly requested via ci:eval-claude label.
+if [ -f /workspace/IS_DOCS_ONLY ] && [ ! -f /workspace/SHOULD_RUN_CLAUDE_CODE ]; then
+  echo "Documentation-only PR without ci:eval-claude label; skipping ${SUT}/${SUITE}."
+  exit 0
+fi
+
 # The smoke-test suite always runs regardless of PR labels; other suites are
 # gated by the ci:eval-claude label (marker written by the preflight step).
 if [ "${SUITE}" != "smoke-test" ] && [ ! -f /workspace/SHOULD_RUN_CLAUDE_CODE ]; then

.ci/run_gemini_cli.sh

@@ -11,6 +11,12 @@ set -e
 SUITE="${1:?usage: run_gemini_cli.sh <suite-name>}"
 SUT="gemini-cli"
 
+# Documentation-only PRs skip smoke tests unless explicitly requested via ci:eval label.
+if [ -f /workspace/IS_DOCS_ONLY ] && [ ! -f /workspace/SHOULD_RUN_GEMINI_CLI ]; then
+  echo "Documentation-only PR without ci:eval label; skipping ${SUT}/${SUITE}."
+  exit 0
+fi
+
 # The smoke-test suite always runs regardless of PR labels; other suites are
 # gated by the ci:eval label (marker written by the preflight step).
 if [ "${SUITE}" != "smoke-test" ] && [ ! -f /workspace/SHOULD_RUN_GEMINI_CLI ]; then

evals/.gitignore

@@ -0,0 +1,3 @@
+results/
+.venv/
+uv.toml

evals/README.md

@@ -4,7 +4,7 @@ This directory contains the evaluation suite for the main functionalities of the
 
 ## Overview
 
-The evaluation uses the [evalbench](https://github.com/GoogleCloudPlatform/evalbench) framework with the Gemini CLI and Claude Code orchestrator to run a set of simulated user tasks against the agent.
+The evaluation uses the [evalbench](https://github.com/GoogleCloudPlatform/evalbench) framework with the various agent harness orchestrators to run a set of simulated user tasks.
 
 ## Configuration Files
 
@@ -33,12 +33,25 @@ The suite evaluates the agent across several dimensions using the following scor
 
 ## How to Run
 
-From the `evals/` directory, execute the evaluation using latest [evalbench](https://github.com/GoogleCloudPlatform/evalbench/releases):
-
+From the `evals/` directory, run the evaluation using `uvx`:
 ```bash
 cd evals/
-evalbench run core-cujs/run_gemini_cli.yaml      # Gemini CLI
-evalbench run core-cujs/run_claude.yaml          # Claude Code
+UV_CONFIG_FILE=uv.toml uvx --index-url https://pypi.org/simple/ google-evalbench --experiment_config=core-cujs/run_gemini_cli.yaml
 ```
 
-Results will be generated in the `results/` directory as CSV reports.
+### Local Execution Caveats
+Unlike CI (which auto-injects values and queries the metadata server), running locally requires these specific alignments:
+
+1.  **Node Version**: Switch your active shell session to Node v20+ before running to support modern regular expressions:
+    ```bash
+    source ~/.nvm/nvm.sh && nvm use 20
+    ```
+2.  **Dirty State Cleanup**: If a run crashes midway, wipe the dirty extension installation before retrying:
+    ```bash
+    rm -rf evals/.venv/fake_home/.gemini/extensions/google-cloud-db-context-engineering
+    ```
+3.  **Environment Variables**: Before executing a run, ensure you have exported your GCP project ID and the global endpoint location:
+    ```bash
+    export GOOGLE_CLOUD_PROJECT="your-gcp-project-id"
+    export GOOGLE_CLOUD_LOCATION="global"
+    ```

evals/model_configs/gemini_model.yaml

@@ -1,4 +1,6 @@
 generator: gcp_vertex_gemini
 vertex_model: gemini-3-flash-preview
+gcp_project_id: "${GOOGLE_CLOUD_PROJECT}"
+gcp_region: "${GOOGLE_CLOUD_LOCATION:global}"
 base_prompt: ""
 execs_per_minute: 5

plugin/skills/autoctx-evaluate/SKILL.md

@@ -68,8 +68,8 @@ Follow these steps exactly in order:
    - You do not need to manually write or extract file contents. Verify that the files have materialized if needed.
 
 4. **Evalbench Run Integration:**
-   - Trigger the `run_shell_command` natively to execute the evaluation from the ROOT of the workspace using the following exact command template:
-     `uvx google-evalbench@1.9.0 --experiment_config=autoctx/experiments/<experiment_name>/eval_configs/run_config.yaml`
+   - Trigger the `run_shell_command` natively to execute the evaluation runner with automatic 3-tiered REST API fallback:
+     `uv run autoctx-eval <experiment_name>`
    - Check the command outputs to ensure the evaluation reports materialize in the respective `autoctx/experiments/<experiment_name>/eval_reports/` directory.
 
 ## Output

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
 
 [project.scripts]
 google-cloud-db-context-engineering = "google.cloud.db_context_enrichment.main:mcp.run"
+autoctx-eval = "google.cloud.db_context_enrichment.evaluate.evaluate_generator:cli_main"
 
 [project.optional-dependencies]
 test = [

src/google/cloud/db_context_enrichment/evaluate/db_generators/base.py

@@ -58,11 +58,17 @@ def generate_model_config(self, context_set_id: str) -> str:
         """
         datasource_ref = self.build_datasource_reference(context_set_id)
 
-        query_context = gda.QueryDataContext(datasource_references=datasource_ref)
-
-        query_context_dict = MessageToDict(
-            query_context._pb, preserving_proto_field_name=True
-        )
+        if isinstance(datasource_ref, dict):
+            query_context_dict = {
+                "datasource_references": datasource_ref
+            }
+        else:
+            query_context = gda.QueryDataContext(
+                datasource_references=datasource_ref
+            )
+            query_context_dict = MessageToDict(
+                query_context._pb, preserving_proto_field_name=True
+            )
 
         model_config = {
             "generator": "query_data_api",
@@ -71,6 +77,12 @@ def generate_model_config(self, context_set_id: str) -> str:
             "context": query_context_dict,
         }
 
+        if self.params.get("use_rest_api"):
+            model_config["use_rest_api"] = True
+
+        if self.params.get("api_endpoint"):
+            model_config["api_endpoint"] = self.params.get("api_endpoint")
+
         return yaml.safe_dump(
             model_config, sort_keys=False, default_flow_style=False
         ).strip()

src/google/cloud/db_context_enrichment/evaluate/evaluate_generator.py

@@ -1,6 +1,8 @@
 import json
+import logging
 import os
 import re
+import subprocess
 import textwrap
 from typing import Any
 
@@ -237,3 +239,120 @@ def _convert_dataset(dataset_path: str, dialect: str) -> str:
         return json.dumps(converted, indent=2)
     except Exception as e:
         raise ValueError(f"Failed to convert dataset at {dataset_path}: {e}")
+
+
+_STAGING_API_ENDPOINT = "staging-geminidataanalytics.sandbox.googleapis.com"
+
+
+_PROTO_FIELD_ERROR_PATTERNS = (
+    "attributeerror",
+    "typeerror",
+    "valueerror",
+    "protocol message",
+    "unknown field",
+    "invalid field",
+    "has no attribute",
+    "querydatacontext",
+    "datasourcereferences",
+)
+
+
+def _exec_evalbench(cmd: list[str]) -> tuple[int, str]:
+    """Executes EvalBench command and returns (returncode, combined_output)."""
+    res = subprocess.run(cmd, capture_output=True, text=True)
+    out = (res.stderr or "") + (res.stdout or "")
+    return res.returncode, out
+
+
+def _is_proto_field_error(output: str) -> bool:
+    """Returns True if the output contains errors related to missing/unreleased proto fields."""
+    out_lower = output.lower()
+    return any(pattern in out_lower for pattern in _PROTO_FIELD_ERROR_PATTERNS)
+
+
+def _update_model_config(
+    model_config_path: str,
+    use_rest_api: bool = True,
+    api_endpoint: str | None = None,
+) -> None:
+    """Updates model_config.yaml with REST API transport flags."""
+    with open(model_config_path) as f:
+        cfg = yaml.safe_load(f) or {}
+
+    cfg["use_rest_api"] = use_rest_api
+    if api_endpoint:
+        cfg["api_endpoint"] = api_endpoint
+
+    with open(model_config_path, "w") as f:
+        yaml.safe_dump(cfg, f, sort_keys=False, default_flow_style=False)
+
+
+def run_evaluation(experiment_name: str) -> None:
+    """
+    Executes EvalBench evaluation for an experiment, using standard SDK gRPC client
+    and falling back to REST (supports nonpublic fields).
+    """
+    logger = logging.getLogger(__name__)
+    eval_configs_dir = f"autoctx/experiments/{experiment_name}/eval_configs"
+    run_config_path = os.path.join(eval_configs_dir, RUN_CONFIG_NAME)
+    model_config_path = os.path.join(eval_configs_dir, MODEL_CONFIG_NAME)
+
+    cmd = [
+        "uvx",
+        "google-evalbench@1.9.0",
+        f"--experiment_config={run_config_path}",
+    ]
+
+    # 1. Standard SDK gRPC Execution
+    logger.info(
+        f"Running EvalBench evaluation for experiment: {experiment_name}"
+    )
+    code, output = _exec_evalbench(cmd)
+    if code == 0 and not _is_proto_field_error(output):
+        logger.info("EvalBench completed successfully via gRPC SDK.")
+        return
+
+    if not _is_proto_field_error(output):
+        logger.error(
+            f"EvalBench execution failed with non-proto error:\n{output[:500]}"
+        )
+        raise RuntimeError(
+            f"EvalBench execution failed with exit code {code}:\n{output}"
+        )
+
+    # 2. REST API Fallback Tiers (Production REST, then Staging REST)
+    rest_tiers = [
+        ("Production REST API", True, None),
+        (f"Staging REST API ({_STAGING_API_ENDPOINT})", True, _STAGING_API_ENDPOINT),
+    ]
+
+    for tier_name, use_rest, endpoint in rest_tiers:
+        logger.info(f"Attempting evaluation fallback via {tier_name}...")
+        try:
+            _update_model_config(
+                model_config_path, use_rest_api=use_rest, api_endpoint=endpoint
+            )
+            code, output = _exec_evalbench(cmd)
+            if code == 0:
+                logger.info(f"Evaluation completed successfully via {tier_name}.")
+                return
+        except Exception as err:
+            logger.debug(f"{tier_name} execution failed: {err}")
+
+    logger.error(f"Evaluation failed across all transport modes:\n{output[:500]}")
+    raise RuntimeError(
+        f"EvalBench evaluation failed for experiment '{experiment_name}'.\n"
+        "You may be attempting to use an unreleased or non-public QueryData feature. Please reach out to your accounts team on for access."
+    )
+
+
+def cli_main() -> None:
+    """CLI entrypoint for autoctx-eval command."""
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: autoctx-eval <experiment_name>")
+        sys.exit(1)
+
+    experiment_name = sys.argv[1]
+    run_evaluation(experiment_name)