Removes hardcoded paths

Author: mafaldasv-dev

.gitignore

@@ -173,7 +173,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 # Abstra
 # Abstra is an AI-powered process automation framework.
@@ -210,4 +210,7 @@ __marimo__/
 # Cursor
 .cursor/
 AGENTS.md
-spec/
+spec/
+
+# macOS
+.DS_Store

examples/opik_vs_elastic/.env.example

@@ -15,10 +15,14 @@ KIBANA_PASSWORD=<PLACEHOLDER>
 CONNECTOR_ID=gemini-25-flash  # alternatives: eis-google-gemini-3-1-pro  # gemini-2.5-flash-preconfigured
 EVALUATION_CONNECTOR_ID=gemini-25-pro  # alternatives: eis-openai-gpt-5-4  # gemini-2.5-pro-preconfigured
 
+# GCS dataset paths. Only needed when USE_GCP = True:
+WIX_QA_DATASET_PATH=gs://<BUCKET>/queries/wix_qa.csv
+WIX_KNOWLEDGE_BASE_PATH=gs://<BUCKET>/knowledge_bases/wix_knowledge_base.csv
+
 # Opik:
 OPIK_API_KEY=<PLACEHOLDER>
-OPIK_PROJECT_NAME=poc-evals-agent-builder
-OPIK_WORKSPACE=context-and-research-playground
+OPIK_PROJECT_NAME=<PLACEHOLDER>
+OPIK_WORKSPACE=<PLACEHOLDER>
 OPIK_URL_OVERRIDE=https://www.comet.com/opik/api
 
 # Tracing / Elastic evals:

examples/opik_vs_elastic/Instructions.md

@@ -4,24 +4,13 @@ This PoC demonstrates how Agent Builder can use the `kbn/evals` Python SDK for d
 management, experiment tracing, and evaluator score ingestion—capabilities previously
 handled through Opik—and how to build custom evaluators.
 
-## 1. Dependencies
-
-`run2.py` uses external Orca evaluators. Clone the `orca` repo as a sibling of
-`elastic-evals-sdk-python/` so the layout is:
-
-```
-── elastic-evals-sdk-python/
-── orca/
-```
-
-## 2. Python environment
+## 1. Python environment
 
 From the SDK repo root:
 
 ```bash
 uv sync --group dev --extra runner --extra poc
 source .venv/bin/activate
-uv pip install --editable ../orca
 ```
 
 Register the venv as a Jupyter kernel. Only needed if your notebook/IDE doesn't pick up `.venv` automatically:
@@ -30,7 +19,7 @@ Register the venv as a Jupyter kernel. Only needed if your notebook/IDE doesn't
 uv run --no-sync python -m ipykernel install --user --name elastic-evals-poc
 ```
 
-## 3. Secrets
+## 2. Secrets
 
 Create `.env` next to `.env.example`:
 
@@ -39,19 +28,18 @@ cp examples/opik_vs_elastic/.env.example examples/opik_vs_elastic/.env
 ```
 
 Set the local URLs, `ELASTICSEARCH_API_KEY`, `KIBANA_API_KEY`, `CONNECTOR_ID`,
-and `EVALUATION_CONNECTOR_ID`. The Opik variables are used when `run2.py` runs
-the tracked external Orca evaluators. Retrieve internal credentials from Vault
-when needed:
+and `EVALUATION_CONNECTOR_ID`. Retrieve internal credentials from Vault when
+needed:
 
 ```bash
 VAULT_ADDR=https://secrets.elastic.co:8200 vault login --method oidc
 ```
 
 The public Hugging Face dataset does not require an API key.
 
-## 4. Data source and sample size
+## 3. Data source and sample size
 
-Set these values near the top of the script before running it:
+Set these values near the top of the script you're running:
 
 ```python
 USE_ENTIRE_DATASET = False
@@ -65,19 +53,30 @@ files. `DATASET_SAMPLE_SIZE` is ignored when `USE_ENTIRE_DATASET` is `True`.
 The entire knowledge base is always indexed.
 
 The available examples and their order may differ between Hugging Face and GCS.
-`run.py` defaults to 10 examples and `run2.py` defaults to 3.
+Both scripts default to 10 examples.
 
-## 5. GCP access
+## 4. GCP access
 
 Only needed when `USE_GCP = True`. Authenticate with your `@elastic.co` account:
 
 ```bash
 gcloud auth application-default login
 ```
 
-## 6. Local stack
+## 5. Local stack
 
-Use a separate terminal for each service and leave it running.
+The steps below run from a local Kibana checkout. If you don't have one already,
+clone [elastic/kibana](https://github.com/elastic/kibana) and bootstrap it:
+
+```bash
+git clone https://github.com/elastic/kibana.git
+cd kibana
+nvm use
+yarn kbn bootstrap
+```
+
+Run the remaining commands from that `kibana` directory, using a separate
+terminal for each service and leaving it running.
 
 In `kibana/config/kibana.dev.yml`, enable evals and OTLP tracing:
 
@@ -100,7 +99,6 @@ uiSettings:
 ### Elasticsearch
 
 ```bash
-cd /Users/mafaldasavelho/Documents/work-repos/kibana-fork/kibana
 nvm use
 yarn es snapshot --license trial
 ```
@@ -118,7 +116,6 @@ curl --user elastic:changeme \
 Start Docker Desktop, then run:
 
 ```bash
-cd /Users/mafaldasavelho/Documents/work-repos/kibana-fork/kibana
 nvm use
 node scripts/edot_collector.js
 ```
@@ -134,7 +131,6 @@ docker ps \
 ### Kibana
 
 ```bash
-cd /Users/mafaldasavelho/Documents/work-repos/kibana-fork/kibana
 nvm use
 node scripts/kibana --dev --verbose
 ```
@@ -149,7 +145,7 @@ curl --silent --show-error \
   http://localhost:5601/dev/api/status
 ```
 
-## 7. Elasticsearch API key
+## 6. Elasticsearch API key
 
 Both scripts authenticate to Elasticsearch and Kibana. Create an API key against
 the local cluster and paste the `encoded` field into both
@@ -164,7 +160,9 @@ This step is optional if both variables already contain a valid key for the
 current cluster. API keys are cluster-specific, so a key from a previous local
 Elasticsearch snapshot returns `401`.
 
-## 8. Run the PoC
+## 7. Run the PoC
+
+Run both scripts from the SDK repo root.
 
 ### `run.py`: managed workflow
 
@@ -173,15 +171,14 @@ Demonstrates the higher-level workflow. It uses
 run SDK-side and custom evaluators, and ingest their scores.
 
 ```bash
-cd /Users/mafaldasavelho/Documents/work-repos/kibana-fork/evals-python-sdk/elastic-evals-sdk-python
 uv run --no-sync python -m examples.opik_vs_elastic.run
 ```
 
 ### `run2.py`: granular workflow
 
 Demonstrates the lower-level workflow without `run_experiment()`. It directly
-coordinates the Dataset, Evaluators, and Score Ingestion APIs, runs the custom
-Document Recall evaluator, and attaches external Orca scores.
+coordinates the Dataset, Evaluators, and Score Ingestion APIs, and runs the
+custom Document Recall evaluator alongside the Kibana ones.
 
 ```bash
 uv run --no-sync python -m examples.opik_vs_elastic.run2

examples/opik_vs_elastic/helpers/data.py

@@ -14,9 +14,10 @@
 
 from examples.opik_vs_elastic.helpers.helpers import (
     GROUND_TRUTH_COLUMN,
-    WIX_KNOWLEDGE_BASE_PATH,
-    WIX_QA_DATASET_PATH,
+    WIX_KNOWLEDGE_BASE_PATH_ENV,
+    WIX_QA_DATASET_PATH_ENV,
     _parse_relevant_doc_ids,  # noqa: PLC2701
+    get_dataset_path,
 )
 
 HUGGING_FACE_DATASET = "Wix/WixQA"
@@ -85,9 +86,9 @@ def _normalize_knowledge_base(dataframe: pd.DataFrame, *, source: str) -> pd.Dat
 def load_wix_data(*, use_gcp: bool) -> tuple[pd.DataFrame, pd.DataFrame]:
     """Load WixQA examples and their knowledge-base corpus."""
     if use_gcp:
-        qa = _normalize_gcs_qa(pd.read_csv(WIX_QA_DATASET_PATH))
+        qa = _normalize_gcs_qa(pd.read_csv(get_dataset_path(WIX_QA_DATASET_PATH_ENV)))
         knowledge_base = _normalize_knowledge_base(
-            pd.read_csv(WIX_KNOWLEDGE_BASE_PATH),
+            pd.read_csv(get_dataset_path(WIX_KNOWLEDGE_BASE_PATH_ENV)),
             source="GCS Wix knowledge base",
         )
     else:

examples/opik_vs_elastic/helpers/helpers.py

@@ -7,24 +7,30 @@
 from __future__ import annotations
 
 import ast
+import os
 from pathlib import Path
 from typing import Any
 
 ENV_PATH = Path(__file__).parent.parent / ".env"
 INDEX_NAME = "wix_knowledge_base"
 SEARCH_TOOL_ID = "wix-knowledge-search"
 GROUND_TRUTH_COLUMN = "gt_customer_support_wix_knowledge_base"
-WIX_QA_DATASET_PATH = "gs://agent-builder-data-science-datasets/queries/wix_qa.csv"
-WIX_KNOWLEDGE_BASE_PATH = (
-    "gs://agent-builder-data-science-datasets/knowledge_bases/cleaned/"
-    "customer_support/wix_knowledge_base/wix_knowledge_base.csv"
-)
+WIX_QA_DATASET_PATH_ENV = "WIX_QA_DATASET_PATH"
+WIX_KNOWLEDGE_BASE_PATH_ENV = "WIX_KNOWLEDGE_BASE_PATH"
 AGENT_ID = "wix-eval-agent"
 AGENT_NAME = "Wix Agent"
 AGENT_INSTRUCTIONS = "Answer questions using the Wix knowledge base."
 SEARCH_TOOL_DESCRIPTION = "Search the Wix knowledge base articles."
 
 
+def get_dataset_path(variable: str) -> str:
+    """Read a GCS dataset path from the environment. Only needed when USE_GCP is True."""
+    value = os.getenv(variable)
+    if not value:
+        raise ValueError(f"Set {variable} in .env to load the Wix data from GCS, or run with USE_GCP = False.")
+    return value
+
+
 def _parse_relevant_doc_ids(value: Any) -> list[str]:
     if value is None:
         return []

examples/opik_vs_elastic/run.py

@@ -196,9 +196,8 @@ async def main() -> None:
     )
 
     # [IMPORTANT] NOTE: if the dataset doesn't exist, then it creates a new one with the given name. If the dataset already exists,
-    # then it's going to cross-check examples and fully rewrite/update accordingly. Example of a response after sending
-    # the request for the same dataset, but with only the first 5 examples:
-    # UpsertDatasetResponse(dataset_id='0b5ee7b6-9f4a-5c66-b196-6b8cc5154eec', added=0, removed=47, unchanged=5)
+    # then it's going to cross-check examples and fully rewrite/update accordingly: re-running with a smaller sample
+    # removes the examples that are no longer present.
 
     upsert_dataset_response = await elastic_evals_client._datasets_client.upsert(
         name=DATASET_NAME,

examples/opik_vs_elastic/run2.py

@@ -13,8 +13,6 @@
 from typing import Any, cast
 
 from dotenv import load_dotenv
-from orca.evaluation.evaluators.retrieval import F1AtK, PrecisionAtK, RecallAtK  # type: ignore[import-untyped]
-from orca.evaluation.evidence.types import EvaluationEvidence, EvidenceArtifact  # type: ignore[import-untyped]
 
 from elastic_evals.api import (
     Environment,
@@ -305,60 +303,6 @@ async def _resolve_subject(
     )
 
 
-def _orca_evidence(document_ids: list[str]) -> EvaluationEvidence:
-    return EvaluationEvidence(
-        artifacts=[
-            EvidenceArtifact(
-                type="resource_list",
-                data={"resources": [{"reference": {"id": document_id}} for document_id in document_ids]},
-            )
-        ]
-    )
-
-
-def _evaluate_with_orca(
-    executed_runs: list[dict[str, Any]],
-) -> list[tuple[dict[str, Any], EvaluationRun]]:
-    scored: list[tuple[dict[str, Any], EvaluationRun]] = []
-    for executed in executed_runs:
-        run_data = executed["data"]
-        retrieved = _extract_retrieved_doc_ids(run_data.output, tool_id=SEARCH_TOOL_ID)
-        relevant = _to_string_list((run_data.metadata or {}).get("relevant_doc_ids"))
-        evidence = _orca_evidence(retrieved)
-        harness = {
-            "name": "agent_builder_converse",
-            "adapter": "run2_task_output",
-            "locator": None,
-            "metadata": {"trace_id": run_data.trace_id},
-        }
-        for metric in (PrecisionAtK(k=3), RecallAtK(k=3), F1AtK(k=3)):
-            score = metric.score(
-                evidence=evidence.model_dump(mode="json"),
-                harness=harness,
-                relevant_doc_ids=relevant,
-            )
-            if isinstance(score, list):
-                raise TypeError(f"Orca evaluator {metric.name} returned multiple scores")
-            scored.append(
-                _scored_run(
-                    executed,
-                    name=f"orca.{score.name}",
-                    result=EvaluationResult(
-                        score=None if score.scoring_failed else float(score.value),
-                        label="unavailable" if score.scoring_failed else None,
-                        explanation=score.reason,
-                        metadata={
-                            "source": "orca",
-                            "scoring_failed": score.scoring_failed,
-                            "k": 3,
-                        },
-                    ),
-                )
-            )
-    print(f"Orca evaluation complete: scores={len(scored)}")
-    return scored
-
-
 async def main() -> None:
     print("[1] Load configuration and prepare the knowledge base")
     load_dotenv(ENV_PATH)
@@ -597,7 +541,7 @@ async def document_recall_runner() -> EvaluationResult:
     }
     connector_id = config.evaluator_connector_id or config.connector_id
 
-    print("[6] Create the experiment with the initial scores")
+    print("[6] Create the experiment and ingest the scores")
     await _export_scores(
         scores_client,
         export_context,
@@ -613,15 +557,6 @@ async def document_recall_runner() -> EvaluationResult:
         scored_runs=document_recall_scores,
     )
 
-    print("[7] Run external Orca evaluators and attach their scores")
-    await _export_scores(
-        scores_client,
-        export_context,
-        label="External Orca scores",
-        evaluator_model_id="orca",
-        scored_runs=_evaluate_with_orca(executed_runs),
-    )
-
     print(f"Finished.\nExperiment '{EXPERIMENT_NAME}' is ready in Kibana.\nExperiment ID: {experiment_id}")
 
 

tests/examples/test_opik_vs_elastic.py

@@ -11,7 +11,6 @@
 import pytest
 
 pytest.importorskip("pandas")
-pytest.importorskip("orca")
 
 from elastic_evals.api import (  # noqa: E402
     Environment,
@@ -241,25 +240,6 @@ def test_granular_score_request_preserves_experiment_name_and_batches_scores() -
     assert all(score.task.trace_id == "1" * 32 for score in payload.scores)
 
 
-def test_granular_orca_scores_are_namespaced_and_use_stored_run() -> None:
-    scored_runs = granular_run._evaluate_with_orca([_executed_run()])
-
-    assert [evaluation.name for _, evaluation in scored_runs] == [
-        "orca.precision-at-3",
-        "orca.recall-at-3",
-        "orca.f1-at-3",
-    ]
-    assert [evaluation.result.score for _, evaluation in scored_runs if evaluation.result] == [
-        pytest.approx(1 / 3),
-        pytest.approx(1 / 2),
-        pytest.approx(0.4),
-    ]
-    assert all(
-        evaluation.result and evaluation.result.metadata and evaluation.result.metadata["source"] == "orca"
-        for _, evaluation in scored_runs
-    )
-
-
 def test_granular_evaluator_configs_use_reference_data_and_llm_connector() -> None:
     definitions = [
         EvaluatorDefinition(