evalbench/datasets/claude-code-tools/example_run_config.yaml at main · GoogleCloudPlatform/evalbench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
############################################################
### Dataset / Eval Items
############################################################
dataset_config: datasets/claude-code-tools/claude-code.evalset.json
dataset_format: agent-format

# Orchestrator Configuration
# `agent` works for both Gemini CLI and Claude Code (alias of `geminicli`).
orchestrator: agent
model_config: datasets/model_configs/claude_code_model.yaml
simulated_user_model_config: datasets/model_configs/gemini_2.5_pro_model.yaml

# Concurrency: number of scenarios to run in parallel.
# Set to 1 for sequential runs (easier to follow logs, avoids session conflicts).
# Default is 10 (matches Gemini CLI).
runners:
  agent_runners: 1

############################################################
### Scorer Related Configs
############################################################
scorers:
  # By default trajectory_matcher drops native/harness-internal tools
  # (Read, Bash, Edit, ToolSearch, ...) from both expected and actual
  # trajectories before scoring. Uncomment the block below to keep them
  # and score native-tool usage too.
  trajectory_matcher: {}
  # trajectory_matcher:
  #   filter_native_tools: false
  goal_completion:
    model_config: datasets/model_configs/gemini_2.5_pro_model.yaml
  behavioral_metrics:
    model_config: datasets/model_configs/gemini_2.5_pro_model.yaml
  parameter_analysis:
    model_config: datasets/model_configs/gemini_2.5_pro_model.yaml
  turn_count: {}
  agent_steps: {}
  end_to_end_latency: {}
  tool_call_latency: {}
  token_consumption: {}
  tokens_processed: {}
  effective_billed_tokens: {}

############################################################
### Reporting Related Configs
############################################################
reporting:
  csv:
    output_directory: 'results'