diff --git a/CLAUDE.md b/CLAUDE.md index 6e270569..cb201522 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ### Development and Testing - **Install dependencies**: `uv sync --dev` (from repo root — `pyproject.toml` is at root) - **Run all tests**: `uv run pytest` (from repo root) -- **Run single test**: `uv run pytest python/tests/core/test_file.py::TestClass::test_method` +- **Run single test**: `uv run pytest python/tests/core/test_agent.py::TestClass::test_method` - **Linting**: `uv run ruff check` - **Format**: `uv run ruff format` - **Fix lint**: `uv run ruff check --fix` @@ -34,7 +34,7 @@ timbal/ │ │ │ ├── tool.py # Tool wrapper │ │ │ ├── tool_set.py # ToolSet ABC for runtime tool resolution │ │ │ ├── mcp.py # MCPServer — MCP servers as tool sources -│ │ │ ├── llm_router.py # Multi-provider LLM dispatch +│ │ │ ├── llm/ # Multi-provider LLM dispatch (router, registry, clients, retry + one module per API) │ │ │ ├── models.py # Model strings + context window lookup │ │ │ └── test_model.py # Offline TestModel for testing │ │ ├── state/ @@ -564,7 +564,7 @@ async with OTelExporter(endpoint="http://localhost:4318") as exporter: ## Testing Strategy -- Tests live in `python/tests/core/` mirroring the package structure +- Tests live under `python/tests/` mirroring the package (`core/`, `core/llm/`, `state/tracing/`, `collectors/`, …) - All async tests use `pytest-asyncio` (mode=AUTO — no `@pytest.mark.asyncio` needed if configured) - Use `TestModel` to avoid API calls in unit tests - `tmp_path` pytest fixture for file-based provider tests diff --git a/benchmarks/agno/results/bench_agent.txt b/benchmarks/agno/results/bench_agent.txt index 08fe7f84..b029eec8 100644 --- a/benchmarks/agno/results/bench_agent.txt +++ b/benchmarks/agno/results/bench_agent.txt @@ -13,82 +13,82 @@ Timbal: ✓ | Agno: ✓ [Phase 1 · Scenario 1] - Timbal latency... p50=786.6 µs - Timbal memory... 58 KB peak - Timbal burst (50 concurrent)... p50=16.97 ms - Timbal burst memory (50 concurrent)... 1897 KB peak - Timbal throughput conc=1... 1418/s - Timbal throughput conc=10... 1634/s - Timbal throughput conc=50... 1711/s - Agno (no tel) latency... p50=1.75 ms - Agno (no tel) memory... 817 KB peak - Agno (no tel) burst (50 concurrent)... p50=35.83 ms - Agno (no tel) burst memory (50 concurrent)... 1954 KB peak - Agno (no tel) throughput conc=1... 540/s - Agno (no tel) throughput conc=10... 711/s - Agno (no tel) throughput conc=50... 528/s + Timbal latency... p50=276.9 µs + Timbal memory... 42 KB peak + Timbal burst (50 concurrent)... p50=228.3 µs + Timbal burst memory (50 concurrent)... 768 KB peak + Timbal throughput conc=1... 3497/s + Timbal throughput conc=10... 3485/s + Timbal throughput conc=50... 3538/s + Agno (no tel) latency... p50=656.0 µs + Agno (no tel) memory... 2878 KB peak + Agno (no tel) burst (50 concurrent)... p50=69.05 ms + Agno (no tel) burst memory (50 concurrent)... 1782 KB peak + Agno (no tel) throughput conc=1... 1184/s + Agno (no tel) throughput conc=10... 1265/s + Agno (no tel) throughput conc=50... 1543/s [Phase 1 · Scenario 2] - Timbal latency... p50=946.9 µs - Timbal memory... 99 KB peak - Timbal burst (30 concurrent)... p50=27.87 ms - Timbal burst memory (30 concurrent)... 2097 KB peak - Timbal throughput conc=1... 673/s - Timbal throughput conc=10... 773/s - Timbal throughput conc=50... 800/s - Agno (no tel) latency... p50=4.05 ms - Agno (no tel) memory... 883 KB peak - Agno (no tel) burst (30 concurrent)... p50=55.14 ms - Agno (no tel) burst memory (30 concurrent)... 2086 KB peak - Agno (no tel) throughput conc=1... 269/s - Agno (no tel) throughput conc=10... 302/s - Agno (no tel) throughput conc=50... 311/s + Timbal latency... p50=285.5 µs + Timbal memory... 70 KB peak + Timbal burst (30 concurrent)... p50=488.8 µs + Timbal burst memory (30 concurrent)... 1120 KB peak + Timbal throughput conc=1... 1117/s + Timbal throughput conc=10... 1767/s + Timbal throughput conc=50... 1812/s + Agno (no tel) latency... p50=1.67 ms + Agno (no tel) memory... 6189 KB peak + Agno (no tel) burst (30 concurrent)... p50=26.41 ms + Agno (no tel) burst memory (30 concurrent)... 2138 KB peak + Agno (no tel) throughput conc=1... 539/s + Agno (no tel) throughput conc=10... 633/s + Agno (no tel) throughput conc=50... 653/s [Phase 1 · Scenario 3] - Timbal latency... p50=749.8 µs - Timbal memory... 96 KB peak - Timbal burst (40 concurrent)... p50=18.95 ms - Timbal burst memory (40 concurrent)... 2570 KB peak - Timbal throughput conc=1... 1140/s - Timbal throughput conc=10... 1167/s - Timbal throughput conc=50... 1201/s - Agno (no tel) latency... p50=3.25 ms - Agno (no tel) memory... 886 KB peak - Agno (no tel) burst (40 concurrent)... p50=67.20 ms - Agno (no tel) burst memory (40 concurrent)... 2654 KB peak - Agno (no tel) throughput conc=1... 263/s - Agno (no tel) throughput conc=10... 291/s - Agno (no tel) throughput conc=50... 310/s + Timbal latency... p50=282.8 µs + Timbal memory... 61 KB peak + Timbal burst (40 concurrent)... p50=8.05 ms + Timbal burst memory (40 concurrent)... 1291 KB peak + Timbal throughput conc=1... 2074/s + Timbal throughput conc=10... 2268/s + Timbal throughput conc=50... 2409/s + Agno (no tel) latency... p50=1.41 ms + Agno (no tel) memory... 5790 KB peak + Agno (no tel) burst (40 concurrent)... p50=28.05 ms + Agno (no tel) burst memory (40 concurrent)... 2998 KB peak + Agno (no tel) throughput conc=1... 665/s + Agno (no tel) throughput conc=10... 653/s + Agno (no tel) throughput conc=50... 765/s Patching agno.api.agent.acreate_agent_run with AsyncMock... Telemetry HTTP call mocked. [Phase 2 · Scenario 1] - Agno (tel mock) latency... p50=1.61 ms - Agno (tel mock) memory... 816 KB peak - Agno (tel mock) burst (50 concurrent)... p50=33.30 ms - Agno (tel mock) burst memory (50 concurrent)... 1953 KB peak - Agno (tel mock) throughput conc=1... 642/s - Agno (tel mock) throughput conc=10... 746/s - Agno (tel mock) throughput conc=50... 801/s + Agno (tel mock) latency... p50=677.3 µs + Agno (tel mock) memory... 2880 KB peak + Agno (tel mock) burst (50 concurrent)... p50=15.28 ms + Agno (tel mock) burst memory (50 concurrent)... 1795 KB peak + Agno (tel mock) throughput conc=1... 1316/s + Agno (tel mock) throughput conc=10... 1344/s + Agno (tel mock) throughput conc=50... 1416/s [Phase 2 · Scenario 2] - Agno (tel mock) latency... p50=3.91 ms - Agno (tel mock) memory... 881 KB peak - Agno (tel mock) burst (30 concurrent)... p50=56.30 ms - Agno (tel mock) burst memory (30 concurrent)... 2087 KB peak - Agno (tel mock) throughput conc=1... 235/s - Agno (tel mock) throughput conc=10... 285/s - Agno (tel mock) throughput conc=50... 302/s + Agno (tel mock) latency... p50=1.95 ms + Agno (tel mock) memory... 6190 KB peak + Agno (tel mock) burst (30 concurrent)... p50=29.07 ms + Agno (tel mock) burst memory (30 concurrent)... 2107 KB peak + Agno (tel mock) throughput conc=1... 552/s + Agno (tel mock) throughput conc=10... 525/s + Agno (tel mock) throughput conc=50... 655/s [Phase 2 · Scenario 3] - Agno (tel mock) latency... p50=3.62 ms - Agno (tel mock) memory... 888 KB peak - Agno (tel mock) burst (40 concurrent)... p50=65.72 ms - Agno (tel mock) burst memory (40 concurrent)... 2657 KB peak - Agno (tel mock) throughput conc=1... 294/s - Agno (tel mock) throughput conc=10... 315/s - Agno (tel mock) throughput conc=50... 328/s + Agno (tel mock) latency... p50=1.45 ms + Agno (tel mock) memory... 5788 KB peak + Agno (tel mock) burst (40 concurrent)... p50=28.32 ms + Agno (tel mock) burst memory (40 concurrent)... 2999 KB peak + Agno (tel mock) throughput conc=1... 638/s + Agno (tel mock) throughput conc=10... 709/s + Agno (tel mock) throughput conc=50... 706/s ──────────────────────────────────────────────────────────────────────────────────────────  Scenario 1: Single tool (LLM → add → LLM → answer) @@ -97,42 +97,42 @@ Latency (×100 sequential runs) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - mean 756.9 µs 1.92 ms 1.59 ms - p50 786.6 µs 1.75 ms 1.61 ms - p95 1.33 ms 3.17 ms 1.93 ms - p99 1.51 ms 8.53 ms 2.02 ms + mean 355.8 µs 780.0 µs 816.4 µs + p50 276.9 µs 656.0 µs 677.3 µs + p95 366.9 µs 825.7 µs 1.02 ms + p99 9.74 ms 10.58 ms 11.01 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 57.6 KB 590 B - Agno (no tel) 816.7 KB 8363 B - Agno (tel mock) 815.8 KB 8354 B + Timbal 42.1 KB 431 B + Agno (no tel) 2878.5 KB 29476 B + Agno (tel mock) 2879.6 KB 29487 B Burst (50 concurrent — fully async on both frameworks) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - p50 16.97 ms 35.83 ms 33.30 ms - p75 17.27 ms 47.06 ms 44.61 ms - p95 17.89 ms 59.00 ms 54.34 ms - p99 17.91 ms 61.86 ms 56.82 ms - max 17.91 ms 61.86 ms 56.82 ms + p50 228.3 µs 69.05 ms 15.28 ms + p75 233.9 µs 120.27 ms 33.42 ms + p95 361.3 µs 131.46 ms 36.48 ms + p99 9.95 ms 140.54 ms 38.41 ms + max 9.95 ms 140.54 ms 38.41 ms - wall: Timbal: 17.9 ms | Agno (no tel): 61.9 ms | Agno (tel mock): 56.8 ms + wall: Timbal: 10.0 ms | Agno (no tel): 140.5 ms | Agno (tel mock): 38.4 ms Burst memory (50 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 1897.2 KB 38855 B - Agno (no tel) 1954.0 KB 40019 B - Agno (tel mock) 1953.3 KB 40003 B + Timbal 767.8 KB 15725 B + Agno (no tel) 1782.1 KB 36497 B + Agno (tel mock) 1795.3 KB 36769 B Throughput (200 loops) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - 1 1418/s 540/s 642/s - 10 1634/s 711/s 746/s - 50 1711/s 528/s 801/s + 1 3497/s 1184/s 1316/s + 10 3485/s 1265/s 1344/s + 50 3538/s 1543/s 1416/s ──────────────────────────────────────────────────────────────────────────────────────────  Scenario 2: Multi-step (LLM → add → LLM → mul → LLM → sub → LLM → answer) @@ -141,42 +141,42 @@ Latency (×100 sequential runs) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - mean 1.02 ms 4.21 ms 3.93 ms - p50 946.9 µs 4.05 ms 3.91 ms - p95 2.24 ms 5.82 ms 4.55 ms - p99 2.92 ms 10.00 ms 4.72 ms + mean 396.7 µs 1.81 ms 2.22 ms + p50 285.5 µs 1.67 ms 1.95 ms + p95 482.4 µs 2.04 ms 3.02 ms + p99 10.26 ms 12.17 ms 15.42 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 99.1 KB 1014 B - Agno (no tel) 883.0 KB 9042 B - Agno (tel mock) 880.6 KB 9017 B + Timbal 69.9 KB 716 B + Agno (no tel) 6189.1 KB 63377 B + Agno (tel mock) 6189.8 KB 63383 B Burst (30 concurrent — fully async on both frameworks) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - p50 27.87 ms 55.14 ms 56.30 ms - p75 29.08 ms 74.74 ms 76.57 ms - p95 29.47 ms 91.17 ms 93.31 ms - p99 29.52 ms 94.71 ms 97.25 ms - max 29.52 ms 94.71 ms 97.25 ms + p50 488.8 µs 26.41 ms 29.07 ms + p75 565.3 µs 35.25 ms 36.39 ms + p95 811.1 µs 49.55 ms 52.40 ms + p99 10.51 ms 50.11 ms 55.03 ms + max 10.51 ms 50.11 ms 55.03 ms - wall: Timbal: 29.5 ms | Agno (no tel): 94.7 ms | Agno (tel mock): 97.2 ms + wall: Timbal: 10.5 ms | Agno (no tel): 50.1 ms | Agno (tel mock): 55.0 ms Burst memory (30 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 2096.7 KB 71567 B - Agno (no tel) 2086.5 KB 71218 B - Agno (tel mock) 2087.4 KB 71250 B + Timbal 1120.5 KB 38245 B + Agno (no tel) 2138.2 KB 72985 B + Agno (tel mock) 2107.5 KB 71936 B Throughput (200 loops) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - 1 673/s 269/s 235/s - 10 773/s 302/s 285/s - 50 800/s 311/s 302/s + 1 1117/s 539/s 552/s + 10 1767/s 633/s 525/s + 50 1812/s 653/s 655/s ──────────────────────────────────────────────────────────────────────────────────────────  Scenario 3: Parallel tools (LLM → [add, mul, neg] concurrent → LLM → answer) @@ -185,42 +185,42 @@ Latency (×100 sequential runs) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - mean 766.2 µs 3.28 ms 3.63 ms - p50 749.8 µs 3.25 ms 3.62 ms - p95 1.24 ms 3.72 ms 4.08 ms - p99 1.53 ms 3.95 ms 4.42 ms + mean 386.5 µs 1.86 ms 1.59 ms + p50 282.8 µs 1.41 ms 1.45 ms + p95 396.5 µs 5.13 ms 1.74 ms + p99 10.26 ms 11.64 ms 11.92 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 95.6 KB 979 B - Agno (no tel) 886.4 KB 9077 B - Agno (tel mock) 888.1 KB 9094 B + Timbal 60.7 KB 621 B + Agno (no tel) 5789.6 KB 59286 B + Agno (tel mock) 5788.2 KB 59271 B Burst (40 concurrent — fully async on both frameworks) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - p50 18.95 ms 67.20 ms 65.72 ms - p75 22.08 ms 92.32 ms 91.09 ms - p95 22.51 ms 113.40 ms 111.53 ms - p99 22.56 ms 116.33 ms 114.90 ms - max 22.56 ms 116.33 ms 114.90 ms + p50 8.05 ms 28.05 ms 28.32 ms + p75 18.00 ms 36.00 ms 36.91 ms + p95 18.18 ms 58.50 ms 60.47 ms + p99 18.37 ms 58.61 ms 61.90 ms + max 18.37 ms 58.61 ms 61.90 ms - wall: Timbal: 22.6 ms | Agno (no tel): 116.3 ms | Agno (tel mock): 114.9 ms + wall: Timbal: 18.4 ms | Agno (no tel): 58.6 ms | Agno (tel mock): 61.9 ms Burst memory (40 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 2570.1 KB 65795 B - Agno (no tel) 2654.4 KB 67953 B - Agno (tel mock) 2657.5 KB 68031 B + Timbal 1290.6 KB 33039 B + Agno (no tel) 2997.5 KB 76736 B + Agno (tel mock) 2999.3 KB 76782 B Throughput (200 loops) Timbal Agno (no tel) Agno (tel mock) ──────────── ──────────────── ──────────────── ──────────────── - 1 1140/s 263/s 294/s - 10 1167/s 291/s 315/s - 50 1201/s 310/s 328/s + 1 2074/s 665/s 638/s + 10 2268/s 653/s 709/s + 50 2409/s 765/s 706/s ────────────────────────────────────────────────────────────────────────────────────────── Timbal: TestModel (sync handler, stateless). Built-in InMemory tracing always on. diff --git a/benchmarks/agno/results/bench_double_fanout.txt b/benchmarks/agno/results/bench_double_fanout.txt index 236a65e4..04442569 100644 --- a/benchmarks/agno/results/bench_double_fanout.txt +++ b/benchmarks/agno/results/bench_double_fanout.txt @@ -6,44 +6,44 @@ ════════════════════════════════════════════════════════════════════════════════════════════ [N=4 per phase] - Timbal trivial... p50=1.82 ms - Timbal async work... p50=4.16 ms - Timbal memory... 100.9 KB peak - Timbal burst (100 concurrent, async work)... p50=121.5 ms - Agno trivial... p50=757.0 µs - Agno async work... p50=3.29 ms - Agno memory... 73.2 KB peak - Agno burst (100 concurrent, async work)... p50=53.2 ms + Timbal trivial... p50=776.8 µs + Timbal async work... p50=3.16 ms + Timbal memory... 68.4 KB peak + Timbal burst (100 concurrent, async work)... p50=48.6 ms + Agno trivial... p50=389.9 µs + Agno async work... p50=2.93 ms + Agno memory... 96.6 KB peak + Agno burst (100 concurrent, async work)... p50=28.0 ms [N=8 per phase] - Timbal trivial... p50=2.69 ms - Timbal async work... p50=5.26 ms - Timbal memory... 162.3 KB peak - Timbal burst (100 concurrent, async work)... p50=301.4 ms - Agno trivial... p50=1.02 ms - Agno async work... p50=3.47 ms - Agno memory... 83.0 KB peak - Agno burst (100 concurrent, async work)... p50=64.3 ms + Timbal trivial... p50=1.59 ms + Timbal async work... p50=3.52 ms + Timbal memory... 95.7 KB peak + Timbal burst (100 concurrent, async work)... p50=85.9 ms + Agno trivial... p50=769.5 µs + Agno async work... p50=2.88 ms + Agno memory... 104.7 KB peak + Agno burst (100 concurrent, async work)... p50=43.4 ms [N=16 per phase] - Timbal trivial... p50=4.99 ms - Timbal async work... p50=5.89 ms - Timbal memory... 277.8 KB peak - Timbal burst (100 concurrent, async work)... p50=524.4 ms - Agno trivial... p50=1.17 ms - Agno async work... p50=3.61 ms - Agno memory... 108.1 KB peak - Agno burst (100 concurrent, async work)... p50=101.5 ms + Timbal trivial... p50=1.72 ms + Timbal async work... p50=3.86 ms + Timbal memory... 159.2 KB peak + Timbal burst (100 concurrent, async work)... p50=160.9 ms + Agno trivial... p50=795.7 µs + Agno async work... p50=3.01 ms + Agno memory... 124.9 KB peak + Agno burst (100 concurrent, async work)... p50=76.5 ms [N=32 per phase] - Timbal trivial... p50=7.49 ms - Timbal async work... p50=8.06 ms - Timbal memory... 512.1 KB peak - Timbal burst (100 concurrent, async work)... p50=1119.0 ms - Agno trivial... p50=1.68 ms - Agno async work... p50=3.99 ms - Agno memory... 150.8 KB peak - Agno burst (100 concurrent, async work)... p50=209.1 ms + Timbal trivial... p50=3.10 ms + Timbal async work... p50=4.72 ms + Timbal memory... 287.9 KB peak + Timbal burst (100 concurrent, async work)... p50=361.7 ms + Agno trivial... p50=1.32 ms + Agno async work... p50=3.49 ms + Agno memory... 159.2 KB peak + Agno burst (100 concurrent, async work)... p50=141.5 ms ────────────────────────────────────────────────────────────────────────────────────────────  Width N=4 per phase @@ -52,35 +52,35 @@ Trivial branches (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 1.87 ms 783.3 µs - p50 1.82 ms 757.0 µs - p95 2.42 ms 971.2 µs - p99 2.52 ms 1.33 ms + mean 906.4 µs 403.0 µs + p50 776.8 µs 389.9 µs + p95 986.3 µs 521.5 µs + p99 11.6 ms 750.5 µs Async work branches (1 ms sleep per branch) (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 4.20 ms 3.66 ms - p50 4.16 ms 3.29 ms - p95 4.75 ms 6.78 ms - p99 5.03 ms 9.42 ms + mean 3.27 ms 2.94 ms + p50 3.16 ms 2.93 ms + p95 3.46 ms 3.31 ms + p99 14.3 ms 3.77 ms Burst (100 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 121.5 ms 53.2 ms - p75 121.8 ms 63.3 ms - p95 122.2 ms 70.9 ms - p99 122.3 ms 72.6 ms - max 122.3 ms 72.6 ms + p50 48.6 ms 28.0 ms + p75 49.0 ms 28.3 ms + p95 60.6 ms 39.4 ms + p99 60.9 ms 39.7 ms + max 60.9 ms 39.7 ms - wall: Timbal: 122.3 ms | Agno: 72.6 ms + wall: Timbal: 60.9 ms | Agno: 39.7 ms Memory (trivial, ×100) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 100.9 KB 1033 B - Agno 73.2 KB 750 B + Timbal 68.4 KB 701 B + Agno 96.6 KB 989 B ────────────────────────────────────────────────────────────────────────────────────────────  Width N=8 per phase @@ -89,35 +89,35 @@ Trivial branches (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 2.74 ms 1.42 ms - p50 2.69 ms 1.02 ms - p95 3.35 ms 1.57 ms - p99 3.95 ms 35.5 ms + mean 1.88 ms 2.91 ms + p50 1.59 ms 769.5 µs + p95 3.29 ms 17.0 ms + p99 12.4 ms 34.8 ms Async work branches (1 ms sleep per branch) (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 8.63 ms 3.47 ms - p50 5.26 ms 3.47 ms - p95 31.0 ms 3.77 ms - p99 99.7 ms 3.95 ms + mean 3.66 ms 2.97 ms + p50 3.52 ms 2.88 ms + p95 4.00 ms 3.35 ms + p99 14.4 ms 6.75 ms Burst (100 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 301.4 ms 64.3 ms - p75 302.5 ms 73.5 ms - p95 303.2 ms 81.2 ms - p99 303.5 ms 83.0 ms - max 303.5 ms 83.0 ms + p50 85.9 ms 43.4 ms + p75 86.5 ms 43.7 ms + p95 97.8 ms 54.4 ms + p99 98.0 ms 54.7 ms + max 98.0 ms 54.7 ms - wall: Timbal: 303.5 ms | Agno: 83.0 ms + wall: Timbal: 98.0 ms | Agno: 54.7 ms Memory (trivial, ×100) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 162.3 KB 1661 B - Agno 83.0 KB 850 B + Timbal 95.7 KB 980 B + Agno 104.7 KB 1073 B ────────────────────────────────────────────────────────────────────────────────────────────  Width N=16 per phase @@ -126,35 +126,35 @@ Trivial branches (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 6.00 ms 1.24 ms - p50 4.99 ms 1.17 ms - p95 10.1 ms 1.56 ms - p99 57.4 ms 3.19 ms + mean 1.85 ms 812.5 µs + p50 1.72 ms 795.7 µs + p95 2.01 ms 935.6 µs + p99 11.9 ms 1.10 ms Async work branches (1 ms sleep per branch) (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 8.91 ms 3.66 ms - p50 5.89 ms 3.61 ms - p95 19.4 ms 4.18 ms - p99 71.8 ms 5.70 ms + mean 4.01 ms 3.03 ms + p50 3.86 ms 3.01 ms + p95 4.34 ms 3.30 ms + p99 15.2 ms 3.36 ms Burst (100 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 524.4 ms 101.5 ms - p75 525.7 ms 110.9 ms - p95 526.8 ms 118.3 ms - p99 527.1 ms 119.8 ms - max 527.1 ms 119.8 ms + p50 160.9 ms 76.5 ms + p75 161.7 ms 76.7 ms + p95 172.6 ms 87.9 ms + p99 172.9 ms 88.0 ms + max 172.9 ms 88.0 ms - wall: Timbal: 527.1 ms | Agno: 119.8 ms + wall: Timbal: 172.9 ms | Agno: 88.0 ms Memory (trivial, ×100) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 277.8 KB 2845 B - Agno 108.1 KB 1107 B + Timbal 159.2 KB 1630 B + Agno 124.9 KB 1278 B ────────────────────────────────────────────────────────────────────────────────────────────  Width N=32 per phase @@ -163,35 +163,35 @@ Trivial branches (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 7.86 ms 1.73 ms - p50 7.49 ms 1.68 ms - p95 8.07 ms 2.08 ms - p99 43.1 ms 2.50 ms + mean 3.40 ms 1.34 ms + p50 3.10 ms 1.32 ms + p95 4.16 ms 1.42 ms + p99 13.6 ms 1.60 ms Async work branches (1 ms sleep per branch) (×100) Timbal Agno ──────────── ────────────── ────────────── - mean 8.96 ms 4.00 ms - p50 8.06 ms 3.99 ms - p95 9.86 ms 4.44 ms - p99 45.1 ms 5.09 ms + mean 4.92 ms 3.53 ms + p50 4.72 ms 3.49 ms + p95 5.20 ms 3.83 ms + p99 16.7 ms 3.99 ms Burst (100 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 1119.0 ms 209.1 ms - p75 1121.4 ms 218.3 ms - p95 1123.3 ms 226.0 ms - p99 1123.7 ms 227.6 ms - max 1123.7 ms 227.6 ms + p50 361.7 ms 141.5 ms + p75 362.5 ms 141.7 ms + p95 363.5 ms 151.6 ms + p99 375.1 ms 151.7 ms + max 375.1 ms 151.7 ms - wall: Timbal: 1123.7 ms | Agno: 227.6 ms + wall: Timbal: 375.1 ms | Agno: 151.7 ms Memory (trivial, ×100) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 512.1 KB 5244 B - Agno 150.8 KB 1544 B + Timbal 287.9 KB 2948 B + Agno 159.2 KB 1631 B ──────────────────────────────────────────────────────────────────────────────────────────── Timbal: two explicit first-class fan-out phases, one task/span per branch. diff --git a/benchmarks/agno/results/bench_parallel.txt b/benchmarks/agno/results/bench_parallel.txt index 9c3194f0..44745984 100644 --- a/benchmarks/agno/results/bench_parallel.txt +++ b/benchmarks/agno/results/bench_parallel.txt @@ -6,54 +6,54 @@ ════════════════════════════════════════════════════════════════════════════════════════ [N=4] - Timbal trivial... p50=1.03 ms - Timbal async work... p50=2.13 ms - Timbal memory... 75.9 KB peak - Timbal burst (200 concurrent, async work)... p50=171.4 ms - Agno trivial... p50=578.6 µs - Agno async work... p50=1.85 ms - Agno memory... 102.5 KB peak - Agno burst (200 concurrent, async work)... p50=62.9 ms + Timbal trivial... p50=439.0 µs + Timbal async work... p50=1.60 ms + Timbal memory... 55.2 KB peak + Timbal burst (200 concurrent, async work)... p50=50.7 ms + Agno trivial... p50=218.4 µs + Agno async work... p50=1.38 ms + Agno memory... 148.4 KB peak + Agno burst (200 concurrent, async work)... p50=26.6 ms [N=8] - Timbal trivial... p50=1.51 ms - Timbal async work... p50=2.55 ms - Timbal memory... 114.4 KB peak - Timbal burst (200 concurrent, async work)... p50=254.4 ms - Agno trivial... p50=746.7 µs - Agno async work... p50=1.99 ms - Agno memory... 113.9 KB peak - Agno burst (200 concurrent, async work)... p50=79.3 ms + Timbal trivial... p50=612.4 µs + Timbal async work... p50=1.76 ms + Timbal memory... 70.0 KB peak + Timbal burst (200 concurrent, async work)... p50=86.8 ms + Agno trivial... p50=295.5 µs + Agno async work... p50=1.41 ms + Agno memory... 156.3 KB peak + Agno burst (200 concurrent, async work)... p50=92.9 ms [N=16] - Timbal trivial... p50=2.49 ms - Timbal async work... p50=3.29 ms - Timbal memory... 187.0 KB peak - Timbal burst (200 concurrent, async work)... p50=544.8 ms - Agno trivial... p50=1.01 ms - Agno async work... p50=2.05 ms - Agno memory... 119.7 KB peak - Agno burst (200 concurrent, async work)... p50=118.9 ms + Timbal trivial... p50=883.9 µs + Timbal async work... p50=1.98 ms + Timbal memory... 106.6 KB peak + Timbal burst (200 concurrent, async work)... p50=155.8 ms + Agno trivial... p50=420.8 µs + Agno async work... p50=1.57 ms + Agno memory... 173.3 KB peak + Agno burst (200 concurrent, async work)... p50=76.5 ms [N=32] - Timbal trivial... p50=4.24 ms - Timbal async work... p50=4.74 ms - Timbal memory... 337.6 KB peak - Timbal burst (200 concurrent, async work)... p50=1224.4 ms - Agno trivial... p50=1.26 ms - Agno async work... p50=2.38 ms - Agno memory... 164.4 KB peak - Agno burst (200 concurrent, async work)... p50=228.0 ms + Timbal trivial... p50=1.43 ms + Timbal async work... p50=2.28 ms + Timbal memory... 185.1 KB peak + Timbal burst (200 concurrent, async work)... p50=310.3 ms + Agno trivial... p50=781.0 µs + Agno async work... p50=1.75 ms + Agno memory... 207.6 KB peak + Agno burst (200 concurrent, async work)... p50=131.7 ms [N=64] - Timbal trivial... p50=7.98 ms - Timbal async work... p50=8.79 ms - Timbal memory... 621.5 KB peak - Timbal burst (200 concurrent, async work)... p50=3153.5 ms - Agno trivial... p50=2.44 ms - Agno async work... p50=3.25 ms - Agno memory... 223.1 KB peak - Agno burst (200 concurrent, async work)... p50=1542.4 ms + Timbal trivial... p50=2.66 ms + Timbal async work... p50=3.20 ms + Timbal memory... 343.7 KB peak + Timbal burst (200 concurrent, async work)... p50=665.2 ms + Agno trivial... p50=1.35 ms + Agno async work... p50=2.35 ms + Agno memory... 277.3 KB peak + Agno burst (200 concurrent, async work)... p50=333.9 ms ────────────────────────────────────────────────────────────────────────────────────────  Width N=4 @@ -62,35 +62,35 @@ Trivial branches (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 1.85 ms 606.5 µs - p50 1.03 ms 578.6 µs - p95 5.60 ms 773.8 µs - p99 17.8 ms 967.7 µs + mean 506.8 µs 224.0 µs + p50 439.0 µs 218.4 µs + p95 554.0 µs 250.0 µs + p99 1.05 ms 289.4 µs Async work branches (1 ms sleep) (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 2.16 ms 1.86 ms - p50 2.13 ms 1.85 ms - p95 2.42 ms 2.03 ms - p99 2.63 ms 2.19 ms + mean 1.88 ms 1.40 ms + p50 1.60 ms 1.38 ms + p95 3.13 ms 1.51 ms + p99 9.88 ms 1.62 ms Burst (200 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 171.4 ms 62.9 ms - p75 175.7 ms 81.6 ms - p95 176.8 ms 96.0 ms - p99 177.1 ms 99.2 ms - max 177.1 ms 99.8 ms + p50 50.7 ms 26.6 ms + p75 51.7 ms 27.4 ms + p95 63.2 ms 28.1 ms + p99 63.4 ms 37.8 ms + max 63.6 ms 37.9 ms - wall: Timbal: 177.1 ms | Agno: 99.8 ms + wall: Timbal: 63.6 ms | Agno: 37.9 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 75.9 KB 389 B - Agno 102.5 KB 525 B + Timbal 55.2 KB 283 B + Agno 148.4 KB 760 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=8 @@ -99,35 +99,35 @@ Trivial branches (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 1.57 ms 756.4 µs - p50 1.51 ms 746.7 µs - p95 1.98 ms 942.5 µs - p99 2.88 ms 1.26 ms + mean 703.4 µs 306.3 µs + p50 612.4 µs 295.5 µs + p95 878.5 µs 364.8 µs + p99 1.09 ms 534.7 µs Async work branches (1 ms sleep) (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 3.01 ms 2.04 ms - p50 2.55 ms 1.99 ms - p95 6.80 ms 2.30 ms - p99 12.6 ms 2.93 ms + mean 1.81 ms 1.45 ms + p50 1.76 ms 1.41 ms + p95 1.96 ms 1.60 ms + p99 2.07 ms 2.35 ms Burst (200 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 254.4 ms 79.3 ms - p75 257.1 ms 99.3 ms - p95 258.2 ms 115.9 ms - p99 258.3 ms 118.8 ms - max 258.7 ms 119.4 ms + p50 86.8 ms 92.9 ms + p75 87.6 ms 170.9 ms + p95 88.6 ms 198.3 ms + p99 99.5 ms 207.0 ms + max 99.6 ms 209.8 ms - wall: Timbal: 258.7 ms | Agno: 119.4 ms + wall: Timbal: 99.6 ms | Agno: 209.8 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 114.4 KB 586 B - Agno 113.9 KB 583 B + Timbal 70.0 KB 359 B + Agno 156.3 KB 800 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=16 @@ -136,35 +136,35 @@ Trivial branches (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 2.49 ms 1.55 ms - p50 2.49 ms 1.01 ms - p95 2.89 ms 4.65 ms - p99 3.07 ms 7.42 ms + mean 978.7 µs 428.0 µs + p50 883.9 µs 420.8 µs + p95 1.15 ms 475.0 µs + p99 1.63 ms 513.9 µs Async work branches (1 ms sleep) (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 3.28 ms 2.05 ms - p50 3.29 ms 2.05 ms - p95 3.68 ms 2.23 ms - p99 3.85 ms 2.33 ms + mean 2.06 ms 1.57 ms + p50 1.98 ms 1.57 ms + p95 2.26 ms 1.69 ms + p99 2.55 ms 1.75 ms Burst (200 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 544.8 ms 118.9 ms - p75 546.8 ms 151.9 ms - p95 548.2 ms 169.5 ms - p99 548.4 ms 172.5 ms - max 548.6 ms 173.1 ms + p50 155.8 ms 76.5 ms + p75 156.9 ms 76.9 ms + p95 158.0 ms 77.1 ms + p99 168.8 ms 87.5 ms + max 168.9 ms 87.7 ms - wall: Timbal: 548.6 ms | Agno: 173.1 ms + wall: Timbal: 168.9 ms | Agno: 87.7 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 187.0 KB 957 B - Agno 119.7 KB 613 B + Timbal 106.6 KB 546 B + Agno 173.3 KB 887 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=32 @@ -173,35 +173,35 @@ Trivial branches (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 4.42 ms 1.25 ms - p50 4.24 ms 1.26 ms - p95 4.72 ms 1.49 ms - p99 4.86 ms 1.65 ms + mean 1.51 ms 771.4 µs + p50 1.43 ms 781.0 µs + p95 1.67 ms 892.8 µs + p99 2.12 ms 1.00 ms Async work branches (1 ms sleep) (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 4.88 ms 2.46 ms - p50 4.74 ms 2.38 ms - p95 5.32 ms 2.66 ms - p99 5.63 ms 6.74 ms + mean 2.41 ms 1.77 ms + p50 2.28 ms 1.75 ms + p95 2.76 ms 1.86 ms + p99 3.02 ms 1.91 ms Burst (200 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 1224.4 ms 228.0 ms - p75 1227.8 ms 245.9 ms - p95 1230.5 ms 260.9 ms - p99 1230.8 ms 263.8 ms - max 1230.8 ms 264.4 ms + p50 310.3 ms 131.7 ms + p75 311.7 ms 132.1 ms + p95 312.7 ms 132.6 ms + p99 323.5 ms 142.5 ms + max 323.7 ms 142.7 ms - wall: Timbal: 1230.8 ms | Agno: 264.4 ms + wall: Timbal: 323.7 ms | Agno: 142.7 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 337.6 KB 1728 B - Agno 164.4 KB 842 B + Timbal 185.1 KB 947 B + Agno 207.6 KB 1063 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=64 @@ -210,35 +210,35 @@ Trivial branches (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 8.60 ms 4.14 ms - p50 7.98 ms 2.44 ms - p95 8.68 ms 9.26 ms - p99 48.3 ms 48.3 ms + mean 2.89 ms 2.00 ms + p50 2.66 ms 1.35 ms + p95 3.42 ms 5.88 ms + p99 13.4 ms 7.24 ms Async work branches (1 ms sleep) (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 9.85 ms 9.37 ms - p50 8.79 ms 3.25 ms - p95 9.78 ms 33.4 ms - p99 53.7 ms 137.3 ms + mean 3.43 ms 2.45 ms + p50 3.20 ms 2.35 ms + p95 3.98 ms 2.66 ms + p99 13.5 ms 2.93 ms Burst (200 concurrent, async work) Timbal Agno ──────────── ────────────── ────────────── - p50 3153.5 ms 1542.4 ms - p75 3179.0 ms 1568.5 ms - p95 3181.3 ms 1597.4 ms - p99 3192.7 ms 1601.0 ms - max 3196.9 ms 1601.7 ms + p50 665.2 ms 333.9 ms + p75 667.2 ms 334.3 ms + p95 669.1 ms 334.4 ms + p99 680.1 ms 344.5 ms + max 680.3 ms 344.7 ms - wall: Timbal: 3196.9 ms | Agno: 1601.7 ms + wall: Timbal: 680.3 ms | Agno: 344.7 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 621.5 KB 3182 B - Agno 223.1 KB 1142 B + Timbal 343.7 KB 1760 B + Agno 277.3 KB 1420 B ──────────────────────────────────────────────────────────────────────────────────────── Timbal: N first-class workflow branches, one task/span per branch. diff --git a/benchmarks/agno/results/bench_workflow.txt b/benchmarks/agno/results/bench_workflow.txt index b1bd6f02..db437bd9 100644 --- a/benchmarks/agno/results/bench_workflow.txt +++ b/benchmarks/agno/results/bench_workflow.txt @@ -9,52 +9,52 @@ Timbal: ✓ | Agno: ✓ [Scenario · sequential] - Timbal latency... p50=1.00 ms - Timbal memory... 57.6 KB peak - Timbal burst (200 concurrent)... p50=94.48 ms - Timbal throughput conc=1... 1070/s - Timbal throughput conc=10... 1636/s - Timbal throughput conc=50... 1527/s - Timbal throughput conc=200... 1510/s - Agno latency... p50=440.2 µs - Agno memory... 98.1 KB peak - Agno burst (200 concurrent)... p50=437.4 µs - Agno throughput conc=1... 2081/s - Agno throughput conc=10... 2190/s - Agno throughput conc=50... 1882/s - Agno throughput conc=200... 1640/s + Timbal latency... p50=351.0 µs + Timbal memory... 40.1 KB peak + Timbal burst (200 concurrent)... p50=27.86 ms + Timbal throughput conc=1... 2578/s + Timbal throughput conc=10... 4357/s + Timbal throughput conc=50... 4849/s + Timbal throughput conc=200... 4916/s + Agno latency... p50=106.6 µs + Agno memory... 133.6 KB peak + Agno burst (200 concurrent)... p50=117.1 µs + Agno throughput conc=1... 7481/s + Agno throughput conc=10... 7497/s + Agno throughput conc=50... 7581/s + Agno throughput conc=200... 7581/s [Scenario · fanout] - Timbal latency... p50=1.10 ms - Timbal memory... 73.3 KB peak - Timbal burst (200 concurrent)... p50=126.37 ms - Timbal throughput conc=1... 870/s - Timbal throughput conc=10... 1111/s - Timbal throughput conc=50... 1135/s - Timbal throughput conc=200... 1066/s - Agno latency... p50=620.1 µs - Agno memory... 101.3 KB peak - Agno burst (200 concurrent)... p50=55.70 ms - Agno throughput conc=1... 1629/s - Agno throughput conc=10... 1827/s - Agno throughput conc=50... 1923/s - Agno throughput conc=200... 1941/s + Timbal latency... p50=397.9 µs + Timbal memory... 50.7 KB peak + Timbal burst (200 concurrent)... p50=41.59 ms + Timbal throughput conc=1... 2237/s + Timbal throughput conc=10... 3697/s + Timbal throughput conc=50... 3614/s + Timbal throughput conc=200... 3729/s + Agno latency... p50=205.0 µs + Agno memory... 146.0 KB peak + Agno burst (200 concurrent)... p50=19.83 ms + Agno throughput conc=1... 4060/s + Agno throughput conc=10... 5595/s + Agno throughput conc=50... 5914/s + Agno throughput conc=200... 5730/s [Scenario · diamond] - Timbal latency... p50=997.7 µs - Timbal memory... 65.7 KB peak - Timbal burst (200 concurrent)... p50=95.19 ms - Timbal throughput conc=1... 1147/s - Timbal throughput conc=10... 1623/s - Timbal throughput conc=50... 1491/s - Timbal throughput conc=200... 1464/s - Agno latency... p50=539.5 µs - Agno memory... 100.4 KB peak - Agno burst (200 concurrent)... p50=58.66 ms - Agno throughput conc=1... 1765/s - Agno throughput conc=10... 2042/s - Agno throughput conc=50... 2064/s - Agno throughput conc=200... 2072/s + Timbal latency... p50=365.0 µs + Timbal memory... 47.0 KB peak + Timbal burst (200 concurrent)... p50=30.97 ms + Timbal throughput conc=1... 2419/s + Timbal throughput conc=10... 4079/s + Timbal throughput conc=50... 3374/s + Timbal throughput conc=200... 4339/s + Agno latency... p50=210.2 µs + Agno memory... 144.2 KB peak + Agno burst (200 concurrent)... p50=16.78 ms + Agno throughput conc=1... 3872/s + Agno throughput conc=10... 5263/s + Agno throughput conc=50... 5498/s + Agno throughput conc=200... 6173/s ────────────────────────────────────────────────────────────────────────────────────────  sequential: A → B → C → D @@ -63,35 +63,35 @@ Latency (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 1.04 ms 461.1 µs - p50 1.00 ms 440.2 µs - p95 1.35 ms 570.9 µs - p99 1.77 ms 694.9 µs + mean 426.2 µs 109.0 µs + p50 351.0 µs 106.6 µs + p95 544.8 µs 124.2 µs + p99 727.9 µs 174.0 µs Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 57.6 KB 295 B - Agno 98.1 KB 502 B + Timbal 40.1 KB 205 B + Agno 133.6 KB 684 B Burst (200 concurrent) Timbal Agno ──────────── ────────────── ────────────── - p50 94.48 ms 437.4 µs - p75 96.05 ms 448.0 µs - p95 97.26 ms 587.0 µs - p99 97.42 ms 754.5 µs - max 97.43 ms 1.78 ms + p50 27.86 ms 117.1 µs + p75 29.72 ms 129.7 µs + p95 42.10 ms 170.6 µs + p99 42.49 ms 323.4 µs + max 42.66 ms 10.31 ms - wall: Timbal: 97.4 ms | Agno: 1.8 ms + wall: Timbal: 42.7 ms | Agno: 10.3 ms Throughput (500 loops) Timbal Agno ──────────── ────────────── ────────────── - 1 1070/s 2081/s - 10 1636/s 2190/s - 50 1527/s 1882/s - 200 1510/s 1640/s + 1 2578/s 7481/s + 10 4357/s 7497/s + 50 4849/s 7581/s + 200 4916/s 7581/s ────────────────────────────────────────────────────────────────────────────────────────  fanout: A → [B, C, D] → E @@ -100,35 +100,35 @@ Latency (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 1.15 ms 668.3 µs - p50 1.10 ms 620.1 µs - p95 1.47 ms 948.5 µs - p99 1.94 ms 1.35 ms + mean 455.5 µs 213.5 µs + p50 397.9 µs 205.0 µs + p95 462.0 µs 270.3 µs + p99 662.6 µs 390.0 µs Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 73.3 KB 375 B - Agno 101.3 KB 518 B + Timbal 50.7 KB 259 B + Agno 146.0 KB 748 B Burst (200 concurrent) Timbal Agno ──────────── ────────────── ────────────── - p50 126.37 ms 55.70 ms - p75 136.99 ms 73.54 ms - p95 137.72 ms 87.92 ms - p99 137.88 ms 90.95 ms - max 137.90 ms 91.53 ms + p50 41.59 ms 19.83 ms + p75 42.70 ms 20.59 ms + p95 54.59 ms 30.83 ms + p99 54.93 ms 30.93 ms + max 55.06 ms 31.05 ms - wall: Timbal: 137.9 ms | Agno: 91.5 ms + wall: Timbal: 55.1 ms | Agno: 31.1 ms Throughput (500 loops) Timbal Agno ──────────── ────────────── ────────────── - 1 870/s 1629/s - 10 1111/s 1827/s - 50 1135/s 1923/s - 200 1066/s 1941/s + 1 2237/s 4060/s + 10 3697/s 5595/s + 50 3614/s 5914/s + 200 3729/s 5730/s ────────────────────────────────────────────────────────────────────────────────────────  diamond: A → [B, C] → D @@ -137,35 +137,35 @@ Latency (×200) Timbal Agno ──────────── ────────────── ────────────── - mean 1.86 ms 566.7 µs - p50 997.7 µs 539.5 µs - p95 6.73 ms 732.2 µs - p99 16.97 ms 857.5 µs + mean 441.3 µs 218.3 µs + p50 365.0 µs 210.2 µs + p95 508.9 µs 287.5 µs + p99 896.7 µs 431.8 µs Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 65.7 KB 336 B - Agno 100.4 KB 514 B + Timbal 47.0 KB 241 B + Agno 144.2 KB 738 B Burst (200 concurrent) Timbal Agno ──────────── ────────────── ────────────── - p50 95.19 ms 58.66 ms - p75 101.43 ms 76.19 ms - p95 102.97 ms 90.17 ms - p99 103.07 ms 92.89 ms - max 103.07 ms 93.42 ms + p50 30.97 ms 16.78 ms + p75 31.97 ms 17.40 ms + p95 42.37 ms 29.26 ms + p99 42.55 ms 29.36 ms + max 42.68 ms 29.50 ms - wall: Timbal: 103.1 ms | Agno: 93.4 ms + wall: Timbal: 42.7 ms | Agno: 29.5 ms Throughput (500 loops) Timbal Agno ──────────── ────────────── ────────────── - 1 1147/s 1765/s - 10 1623/s 2042/s - 50 1491/s 2064/s - 200 1464/s 2072/s + 1 2419/s 3872/s + 10 4079/s 5263/s + 50 3374/s 5498/s + 200 4339/s 6173/s ──────────────────────────────────────────────────────────────────────────────────────── Timbal: Workflow DAG with built-in InMemory tracing always on. diff --git a/benchmarks/crewai/results/bench_agent.txt b/benchmarks/crewai/results/bench_agent.txt index 708c5a08..d6222a41 100644 --- a/benchmarks/crewai/results/bench_agent.txt +++ b/benchmarks/crewai/results/bench_agent.txt @@ -1,3 +1,4 @@ +Installed 61 packages in 112ms ══════════════════════════════════════════════════════════════════════════════════════════════════════════════  Timbal vs CrewAI — agent loop benchmark @@ -9,127 +10,390 @@ ══════════════════════════════════════════════════════════════════════════════════════════════════════════════ Spot-checking correctness (Scenario 1)... + + Timbal: ✓ | CA sync: ✓ | CA async: ✓ [Phase 1 · Scenario 1] - Timbal latency... p50=978.2 µs - Timbal memory... 59 KB peak - Timbal burst (30 concurrent)... p50=9.51 ms - Timbal burst memory (30 concurrent)... 1121 KB peak - Timbal throughput conc=1... 1692/s - Timbal throughput conc=5... 1952/s - Timbal throughput conc=20... 1935/s - CA sync latency... p50=3.19 ms - CA sync memory... 986 KB peak - CA sync burst (30 bounded, conc≤5)... p50=869.97 ms - CA sync burst memory (30 bounded, conc≤5)... 812 KB peak - CA sync throughput conc=1... 31/s - CA sync throughput conc=5... 15/s - CA sync throughput conc=20... 17/s - CA async latency... p50=3.20 ms - CA async memory... 987 KB peak - CA async burst (30 concurrent)... p50=498.83 ms - CA async burst memory (30 concurrent)... 1724 KB peak - CA async throughput conc=1... 31/s - CA async throughput conc=5... 31/s - CA async throughput conc=20... 30/s + Timbal latency... p50=309.0 µs + Timbal memory... 46 KB peak + Timbal burst (30 concurrent)... p50=244.2 µs + Timbal burst memory (30 concurrent)... 461 KB peak + Timbal throughput conc=1... 3566/s + Timbal throughput conc=5... 2935/s + Timbal throughput conc=20... 3599/s + CA sync latency... + + +p50=8.69 ms + CA sync memory... + + +41885 KB peak + CA sync burst (30 bounded, conc≤5)... + + +p50=790.09 ms + CA sync burst memory (30 bounded, conc≤5)... + + +6767 KB peak + CA sync throughput conc=1... + + +24/s + CA sync throughput conc=5... + + +22/s + CA sync throughput conc=20... + + +18/s + CA async latency... + + +p50=6.74 ms + CA async memory... + + +41920 KB peak + + + CA async burst (30 concurrent)... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +p50=758.06 ms + CA async burst memory (30 concurrent)... + + +2195 KB peak + CA async throughput conc=1... + + +24/s + CA async throughput conc=5... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +26/s + CA async throughput conc=20... + + +26/s [Phase 1 · Scenario 2] - Timbal latency... p50=1.04 ms - Timbal memory... 98 KB peak - Timbal burst (20 concurrent)... p50=19.44 ms - Timbal burst memory (20 concurrent)... 1385 KB peak - Timbal throughput conc=1... 383/s - Timbal throughput conc=5... 727/s - Timbal throughput conc=20... 788/s - CA sync latency... p50=4.19 ms - CA sync memory... 1340 KB peak - CA sync burst (20 bounded, conc≤5)... p50=761.79 ms - CA sync burst memory (20 bounded, conc≤5)... 930 KB peak - CA sync throughput conc=1... 31/s - CA sync throughput conc=5... 13/s - CA sync throughput conc=20... 16/s - CA async latency... p50=3.22 ms - CA async memory... 1285 KB peak - CA async burst (20 concurrent)... p50=347.46 ms - CA async burst memory (20 concurrent)... 1496 KB peak - CA async throughput conc=1... 30/s - CA async throughput conc=5... 32/s - CA async throughput conc=20... 26/s + Timbal latency... p50=325.3 µs + Timbal memory... 87 KB peak + Timbal burst (20 concurrent)... p50=611.5 µs + Timbal burst memory (20 concurrent)... 745 KB peak + Timbal throughput conc=1... 1670/s + Timbal throughput conc=5... 1690/s + Timbal throughput conc=20... 1514/s + CA sync latency... + + +p50=6.99 ms + CA sync memory... + + +55805 KB peak + + + CA sync burst (20 bounded, conc≤5)... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +p50=669.59 ms + CA sync burst memory (20 bounded, conc≤5)... + + +1999 KB peak + CA sync throughput conc=1... + + +25/s + CA sync throughput conc=5... + + +17/s + CA sync throughput conc=20... + + +22/s + CA async latency... + + +p50=6.59 ms + CA async memory... + + +55927 KB peak + CA async burst (20 concurrent)... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +p50=462.38 ms + CA async burst memory (20 concurrent)... + + +2295 KB peak + CA async throughput conc=1... + + +22/s + CA async throughput conc=5... + + +24/s + CA async throughput conc=20... + + +25/s [Phase 1 · Scenario 3] - Timbal latency... p50=964.0 µs - Timbal memory... 89 KB peak - Timbal burst (25 concurrent)... p50=11.92 ms - Timbal burst memory (25 concurrent)... 1584 KB peak - Timbal throughput conc=1... 1151/s - Timbal throughput conc=5... 1206/s - Timbal throughput conc=20... 1290/s - CA sync latency... p50=3.70 ms - CA sync memory... 1303 KB peak - CA sync burst (25 bounded, conc≤5)... p50=694.84 ms - CA sync burst memory (25 bounded, conc≤5)... 844 KB peak - CA sync throughput conc=1... 30/s - CA sync throughput conc=5... 16/s - CA sync throughput conc=20... 19/s - CA async latency... p50=3.27 ms - CA async memory... 1251 KB peak - CA async burst (25 concurrent)... p50=445.03 ms - CA async burst memory (25 concurrent)... 1865 KB peak - CA async throughput conc=1... 28/s - CA async throughput conc=5... 30/s - CA async throughput conc=20... 31/s + Timbal latency... p50=339.5 µs + Timbal memory... 70 KB peak + Timbal burst (25 concurrent)... p50=6.36 ms + Timbal burst memory (25 concurrent)... 805 KB peak + Timbal throughput conc=1... 1907/s + Timbal throughput conc=5... 2100/s + Timbal throughput conc=20... 1974/s + CA sync latency... + + +p50=7.32 ms + CA sync memory... + + +54612 KB peak + CA sync burst (25 bounded, conc≤5)... + + +p50=710.89 ms + CA sync burst memory (25 bounded, conc≤5)... + + +2316 KB peak + CA sync throughput conc=1... + + +24/s + CA sync throughput conc=5... + + +21/s + CA sync throughput conc=20... + + +14/s + CA async latency... + + +p50=6.91 ms + CA async memory... + + +54309 KB peak + CA async burst (25 concurrent)... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +p50=546.19 ms + CA async burst memory (25 concurrent)... + + +2843 KB peak + CA async throughput conc=1... + + +24/s + CA async throughput conc=5... + + +23/s + CA async throughput conc=20... + + +25/s Activating AgentOps instrumentation... AgentOps instrumented. [Phase 2 · Scenario 1] - CA sync+AO latency... p50=4.40 ms - CA sync+AO memory... 20272 KB peak - CA sync+AO burst (30 bounded, conc≤5)... p50=880.31 ms - CA sync+AO burst memory (30 bounded, conc≤5)... 1169 KB peak - CA sync+AO throughput conc=1... 30/s - CA sync+AO throughput conc=5... 15/s - CA sync+AO throughput conc=20... 20/s - CA async+AO latency... p50=3.05 ms - CA async+AO memory... 988 KB peak - CA async+AO burst (30 concurrent)... p50=502.52 ms - CA async+AO burst memory (30 concurrent)... 1726 KB peak - CA async+AO throughput conc=1... 31/s - CA async+AO throughput conc=5... 31/s - CA async+AO throughput conc=20... 31/s + CA sync+AO latency... + + +p50=9.58 ms + CA sync+AO memory... + + +49690 KB peak + CA sync+AO burst (30 bounded, conc≤5)... + + +p50=810.82 ms + CA sync+AO burst memory (30 bounded, conc≤5)... + + +1980 KB peak + + CA sync+AO throughput conc=1... + + +24/s + CA sync+AO throughput conc=5... + + +18/s + CA sync+AO throughput conc=20... + + +18/s + CA async+AO latency... + + +p50=6.88 ms + CA async+AO memory... + + +41873 KB peak + CA async+AO burst (30 concurrent)... + + +p50=630.24 ms + CA async+AO burst memory (30 concurrent)... + + +2200 KB peak + CA async+AO throughput conc=1... + + +25/s + CA async+AO throughput conc=5... + + +23/s + CA async+AO throughput conc=20... + + +25/s [Phase 2 · Scenario 2] - CA sync+AO latency... p50=4.80 ms - CA sync+AO memory... 29998 KB peak - CA sync+AO burst (20 bounded, conc≤5)... p50=769.41 ms - CA sync+AO burst memory (20 bounded, conc≤5)... 1539 KB peak - CA sync+AO throughput conc=1... 27/s - CA sync+AO throughput conc=5... 16/s - CA sync+AO throughput conc=20... 18/s - CA async+AO latency... p50=3.44 ms - CA async+AO memory... 1283 KB peak - CA async+AO burst (20 concurrent)... p50=367.45 ms - CA async+AO burst memory (20 concurrent)... 1498 KB peak - CA async+AO throughput conc=1... 29/s - CA async+AO throughput conc=5... 30/s - CA async+AO throughput conc=20... 30/s + CA sync+AO latency... + + +p50=8.58 ms + CA sync+AO memory... + + +65077 KB peak + CA sync+AO burst (20 bounded, conc≤5)... + + +p50=691.76 ms + CA sync+AO burst memory (20 bounded, conc≤5)... + + +2189 KB peak + CA sync+AO throughput conc=1... + + +22/s + CA sync+AO throughput conc=5... + + +26/s + CA sync+AO throughput conc=20... + + +21/s + CA async+AO latency... + + +p50=8.17 ms + CA async+AO memory... + + +55877 KB peak + CA async+AO burst (20 concurrent)... + + +p50=474.34 ms + CA async+AO burst memory (20 concurrent)... + + +2287 KB peak + CA async+AO throughput conc=1... + + +21/s + CA async+AO throughput conc=5... + + +20/s + CA async+AO throughput conc=20... + + +23/s [Phase 2 · Scenario 3] - CA sync+AO latency... p50=4.87 ms - CA sync+AO memory... 28966 KB peak - CA sync+AO burst (25 bounded, conc≤5)... p50=844.18 ms - CA sync+AO burst memory (25 bounded, conc≤5)... 1502 KB peak - CA sync+AO throughput conc=1... 29/s - CA sync+AO throughput conc=5... 17/s - CA sync+AO throughput conc=20... 21/s - CA async+AO latency... p50=3.36 ms - CA async+AO memory... 1250 KB peak - CA async+AO burst (25 concurrent)... p50=433.60 ms - CA async+AO burst memory (25 concurrent)... 1862 KB peak - CA async+AO throughput conc=1... 28/s - CA async+AO throughput conc=5... 30/s - CA async+AO throughput conc=20... 30/s + CA sync+AO latency... + + +p50=11.07 ms + + + CA sync+AO memory... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +63735 KB peak + CA sync+AO burst (25 bounded, conc≤5)... + + +p50=871.71 ms + CA sync+AO burst memory (25 bounded, conc≤5)... + + +2997 KB peak + CA sync+AO throughput conc=1... + + +21/s + CA sync+AO throughput conc=5... + + +19/s + CA sync+AO throughput conc=20... + + +18/s + CA async+AO latency... + + +p50=8.30 ms + CA async+AO memory... + + +54288 KB peak + + + CA async+AO burst (25 concurrent)... ╭────────────────────────── Tracing Preference Saved ──────────────────────────╮ + + +p50=590.58 ms + CA async+AO burst memory (25 concurrent)... + + +2853 KB peak + CA async+AO throughput conc=1... + + +21/s + CA async+AO throughput conc=5... + + +23/s + CA async+AO throughput conc=20... + + +23/s ──────────────────────────────────────────────────────────────────────────────────────────────────────────────  Scenario 1: Single tool (LLM → add → LLM → answer) @@ -138,46 +402,46 @@ Latency (×100 Timbal async · CA sync: sequential · CA async: sequential) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - mean 953.9 µs 3.32 ms 3.63 ms 4.74 ms 3.09 ms - p50 978.2 µs 3.19 ms 3.20 ms 4.40 ms 3.05 ms - p95 1.58 ms 4.58 ms 5.99 ms 6.50 ms 4.03 ms - p99 1.91 ms 5.53 ms 22.28 ms 12.70 ms 4.45 ms + mean 322.4 µs 11.13 ms 6.98 ms 10.32 ms 7.19 ms + p50 309.0 µs 8.69 ms 6.74 ms 9.58 ms 6.88 ms + p95 570.0 µs 28.16 ms 9.08 ms 14.85 ms 9.62 ms + p99 796.7 µs 70.98 ms 10.04 ms 22.42 ms 13.03 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 58.6 KB 600 B - CA (sync) 986.4 KB 10101 B - CA (async) 987.3 KB 10110 B - CA sync+AO 20272.3 KB 207588 B - CA async+AO 987.5 KB 10112 B + Timbal 46.2 KB 473 B + CA (sync) 41885.5 KB 428907 B + CA (async) 41919.6 KB 429256 B + CA sync+AO 49690.2 KB 508828 B + CA async+AO 41873.3 KB 428783 B Burst (30 concurrent — Timbal/CA async: unbounded · CA sync: bounded conc≤5) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - p50 9.51 ms 869.97 ms 498.83 ms 880.31 ms 502.52 ms - p75 9.64 ms 1046.63 ms 696.11 ms 1065.39 ms 697.84 ms - p95 9.67 ms 1181.97 ms 864.15 ms 1326.15 ms 870.23 ms - p99 9.88 ms 1441.93 ms 893.09 ms 1570.71 ms 899.45 ms - max 9.88 ms 1441.93 ms 893.09 ms 1570.71 ms 899.45 ms + p50 244.2 µs 790.09 ms 758.06 ms 810.82 ms 630.24 ms + p75 261.7 µs 1024.46 ms 1093.54 ms 1037.18 ms 870.59 ms + p95 326.5 µs 1432.74 ms 1322.67 ms 1454.68 ms 1084.18 ms + p99 472.3 µs 1434.27 ms 1355.73 ms 1457.56 ms 1125.17 ms + max 472.3 µs 1434.27 ms 1355.73 ms 1457.56 ms 1125.17 ms - wall: Timbal: 9.9 ms | CA (sync): 1441.9 ms | CA (async): 893.1 ms | CA sync+AO: 1570.7 ms | CA async+AO: 899.4 ms + wall: Timbal: 0.5 ms | CA (sync): 1434.3 ms | CA (async): 1355.7 ms | CA sync+AO: 1457.6 ms | CA async+AO: 1125.2 ms Burst memory (30 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 1121.3 KB 38274 B - CA (sync) 811.6 KB 27701 B - CA (async) 1724.1 KB 58848 B - CA sync+AO 1168.7 KB 39893 B - CA async+AO 1726.1 KB 58919 B + Timbal 461.5 KB 15752 B + CA (sync) 6766.8 KB 230974 B + CA (async) 2194.7 KB 74911 B + CA sync+AO 1979.7 KB 67574 B + CA async+AO 2200.1 KB 75096 B Throughput (100 loops · CA sync capped at conc≤5) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - 1 1692/s 31/s 31/s 30/s 31/s - 5 1952/s 15/s 31/s 15/s 31/s - 20 1935/s 17/s 30/s 20/s 31/s + 1 3566/s 24/s 24/s 24/s 25/s + 5 2935/s 22/s 26/s 18/s 23/s + 20 3599/s 18/s 26/s 18/s 25/s ──────────────────────────────────────────────────────────────────────────────────────────────────────────────  Scenario 2: Multi-step (LLM → add → LLM → mul → LLM → sub → LLM → answer) @@ -186,46 +450,46 @@ Latency (×100 Timbal async · CA sync: sequential · CA async: sequential) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - mean 1.38 ms 4.49 ms 3.25 ms 4.94 ms 3.50 ms - p50 1.04 ms 4.19 ms 3.22 ms 4.80 ms 3.44 ms - p95 1.63 ms 7.14 ms 4.11 ms 6.57 ms 4.62 ms - p99 38.87 ms 8.87 ms 4.89 ms 7.82 ms 6.09 ms + mean 337.3 µs 7.40 ms 6.98 ms 9.55 ms 8.70 ms + p50 325.3 µs 6.99 ms 6.59 ms 8.58 ms 8.17 ms + p95 502.5 µs 9.78 ms 9.48 ms 13.74 ms 11.69 ms + p99 848.7 µs 10.41 ms 11.44 ms 16.93 ms 12.47 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 98.0 KB 1003 B - CA (sync) 1339.6 KB 13717 B - CA (async) 1285.0 KB 13158 B - CA sync+AO 29997.6 KB 307176 B - CA async+AO 1283.3 KB 13141 B + Timbal 86.6 KB 886 B + CA (sync) 55805.2 KB 571445 B + CA (async) 55926.8 KB 572691 B + CA sync+AO 65077.0 KB 666388 B + CA async+AO 55876.8 KB 572179 B Burst (20 concurrent — Timbal/CA async: unbounded · CA sync: bounded conc≤5) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - p50 19.44 ms 761.79 ms 347.46 ms 769.41 ms 367.45 ms - p75 19.50 ms 867.24 ms 491.28 ms 897.76 ms 511.82 ms - p95 19.55 ms 1222.68 ms 599.78 ms 1211.76 ms 627.04 ms - p99 19.55 ms 1222.68 ms 599.78 ms 1211.76 ms 627.04 ms - max 19.55 ms 1222.68 ms 599.78 ms 1211.76 ms 627.04 ms + p50 611.5 µs 669.59 ms 462.38 ms 691.76 ms 474.34 ms + p75 709.0 µs 845.97 ms 636.45 ms 848.49 ms 647.51 ms + p95 918.2 µs 1570.10 ms 789.65 ms 914.86 ms 787.46 ms + p99 918.2 µs 1570.10 ms 789.65 ms 914.86 ms 787.46 ms + max 918.2 µs 1570.10 ms 789.65 ms 914.86 ms 787.46 ms - wall: Timbal: 19.6 ms | CA (sync): 1222.7 ms | CA (async): 599.8 ms | CA sync+AO: 1211.8 ms | CA async+AO: 627.0 ms + wall: Timbal: 0.9 ms | CA (sync): 1570.1 ms | CA (async): 789.6 ms | CA sync+AO: 914.9 ms | CA async+AO: 787.5 ms Burst memory (20 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 1384.7 KB 70897 B - CA (sync) 929.6 KB 47595 B - CA (async) 1496.4 KB 76615 B - CA sync+AO 1539.4 KB 78815 B - CA async+AO 1498.0 KB 76697 B + Timbal 745.2 KB 38155 B + CA (sync) 1998.7 KB 102335 B + CA (async) 2294.5 KB 117480 B + CA sync+AO 2188.6 KB 112054 B + CA async+AO 2287.0 KB 117095 B Throughput (100 loops · CA sync capped at conc≤5) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - 1 383/s 31/s 30/s 27/s 29/s - 5 727/s 13/s 32/s 16/s 30/s - 20 788/s 16/s 26/s 18/s 30/s + 1 1670/s 25/s 22/s 22/s 21/s + 5 1690/s 17/s 24/s 26/s 20/s + 20 1514/s 22/s 25/s 21/s 23/s ──────────────────────────────────────────────────────────────────────────────────────────────────────────────  Scenario 3: Parallel tools (Timbal: concurrent | CrewAI: sequential ReAct) @@ -235,46 +499,46 @@ Latency (×100 Timbal async · CA sync: sequential · CA async: sequential) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - mean 1.45 ms 3.86 ms 3.36 ms 5.14 ms 3.41 ms - p50 964.0 µs 3.70 ms 3.27 ms 4.87 ms 3.36 ms - p95 1.91 ms 5.61 ms 4.67 ms 8.49 ms 4.56 ms - p99 43.90 ms 6.13 ms 5.35 ms 12.54 ms 5.34 ms + mean 836.9 µs 7.80 ms 7.46 ms 11.19 ms 8.89 ms + p50 339.5 µs 7.32 ms 6.91 ms 11.07 ms 8.30 ms + p95 565.7 µs 10.63 ms 9.90 ms 14.21 ms 11.80 ms + p99 49.49 ms 12.69 ms 10.49 ms 17.39 ms 15.12 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 89.1 KB 912 B - CA (sync) 1303.2 KB 13345 B - CA (async) 1250.8 KB 12808 B - CA sync+AO 28966.3 KB 296615 B - CA async+AO 1250.0 KB 12800 B + Timbal 69.7 KB 713 B + CA (sync) 54611.8 KB 559225 B + CA (async) 54309.4 KB 556129 B + CA sync+AO 63734.8 KB 652645 B + CA async+AO 54288.0 KB 555909 B Burst (25 concurrent — Timbal/CA async: unbounded · CA sync: bounded conc≤5) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - p50 11.92 ms 694.84 ms 445.03 ms 844.18 ms 433.60 ms - p75 12.88 ms 1056.93 ms 619.23 ms 1185.30 ms 607.15 ms - p95 12.98 ms 1634.61 ms 763.15 ms 1434.84 ms 751.30 ms - p99 12.99 ms 1891.11 ms 792.31 ms 1693.25 ms 780.73 ms - max 12.99 ms 1891.11 ms 792.31 ms 1693.25 ms 780.73 ms + p50 6.36 ms 710.89 ms 546.19 ms 871.71 ms 590.58 ms + p75 6.39 ms 919.05 ms 739.31 ms 1397.35 ms 800.99 ms + p95 6.42 ms 1025.75 ms 905.16 ms 1911.61 ms 973.84 ms + p99 6.43 ms 1260.26 ms 939.88 ms 2025.10 ms 1009.22 ms + max 6.43 ms 1260.26 ms 939.88 ms 2025.10 ms 1009.22 ms - wall: Timbal: 13.0 ms | CA (sync): 1891.1 ms | CA (async): 792.3 ms | CA sync+AO: 1693.2 ms | CA async+AO: 780.7 ms + wall: Timbal: 6.4 ms | CA (sync): 1260.3 ms | CA (async): 939.9 ms | CA sync+AO: 2025.1 ms | CA async+AO: 1009.2 ms Burst memory (25 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 1583.6 KB 64865 B - CA (sync) 843.5 KB 34551 B - CA (async) 1864.8 KB 76381 B - CA sync+AO 1501.7 KB 61508 B - CA async+AO 1862.3 KB 76279 B + Timbal 805.4 KB 32989 B + CA (sync) 2315.8 KB 94855 B + CA (async) 2842.8 KB 116442 B + CA sync+AO 2996.8 KB 122750 B + CA async+AO 2853.5 KB 116878 B Throughput (100 loops · CA sync capped at conc≤5) Timbal CA (sync) CA (async) CA sync+AO CA async+AO ──────────── ────────────── ────────────── ────────────── ────────────── ────────────── - 1 1151/s 30/s 28/s 29/s 28/s - 5 1206/s 16/s 30/s 17/s 30/s - 20 1290/s 19/s 31/s 21/s 30/s + 1 1907/s 24/s 24/s 21/s 21/s + 5 2100/s 21/s 23/s 19/s 23/s + 20 1974/s 14/s 25/s 18/s 23/s ────────────────────────────────────────────────────────────────────────────────────────────────────────────── Timbal: TestModel (sync handler, stateless). Built-in InMemory tracing always on. diff --git a/benchmarks/crewai/results/bench_double_fanout.txt b/benchmarks/crewai/results/bench_double_fanout.txt index aa9643e4..e02d5461 100644 --- a/benchmarks/crewai/results/bench_double_fanout.txt +++ b/benchmarks/crewai/results/bench_double_fanout.txt @@ -1,3 +1,4 @@ +Installed 61 packages in 123ms ════════════════════════════════════════════════════════════════════════════════════════  Timbal vs CrewAI Flow — Double Fan-out @@ -21,55 +22,55 @@ Latency p50 (×200 sequential runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 3.92 ms 6.09 ms - 32 67 7.53 ms 11.8 ms - 64 131 14.4 ms 21.8 ms - 128 259 33.2 ms 52.9 ms + 16 35 1.85 ms 6.77 ms + 32 67 3.78 ms 11.3 ms + 64 131 8.27 ms 23.7 ms + 128 259 20.8 ms 58.7 ms Latency p95 (×200 sequential runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 4.67 ms 7.14 ms - 32 67 8.65 ms 19.9 ms - 64 131 16.6 ms 23.4 ms - 128 259 124.0 ms 56.2 ms + 16 35 2.13 ms 7.81 ms + 32 67 4.60 ms 12.8 ms + 64 131 8.89 ms 25.1 ms + 128 259 31.4 ms 61.7 ms Latency p99 (×200 sequential runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 5.17 ms 8.43 ms - 32 67 9.62 ms 59.5 ms - 64 131 103.1 ms 71.0 ms - 128 259 161.7 ms 134.3 ms + 16 35 2.31 ms 8.36 ms + 32 67 4.87 ms 13.4 ms + 64 131 74.9 ms 35.1 ms + 128 259 103.6 ms 133.6 ms Overhead per extra branch-pair (slope of p50 vs N, 2 branches per unit) Each N increment adds 2 async branches (one per phase) - Timbal +262.0 µs / branch-pair (+131.0 µs per branch) - Flow steps +419.3 µs / branch-pair (+209.7 µs per branch) + Timbal +171.6 µs / branch-pair (+85.8 µs per branch) + Flow steps +471.6 µs / branch-pair (+235.8 µs per branch) Burst p50 (100 concurrent runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 504.2 ms 704.3 ms - 32 67 1014.1 ms 1021.4 ms - 64 131 2214.4 ms 2198.1 ms - 128 259 4838.4 ms 5329.4 ms + 16 35 377.4 ms 483.3 ms + 32 67 487.1 ms 1015.3 ms + 64 131 1156.9 ms 2270.6 ms + 128 259 2539.8 ms 5836.5 ms Burst p95 (100 concurrent runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 506.6 ms 788.9 ms - 32 67 1018.3 ms 1125.1 ms - 64 131 2221.5 ms 2324.7 ms - 128 259 4944.3 ms 5593.3 ms + 16 35 379.9 ms 567.3 ms + 32 67 491.1 ms 1114.4 ms + 64 131 1165.3 ms 2402.6 ms + 128 259 2632.3 ms 6103.8 ms Burst p99 (100 concurrent runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 506.8 ms 798.2 ms - 32 67 1019.0 ms 1134.8 ms - 64 131 2222.2 ms 2335.8 ms - 128 259 4945.7 ms 5607.1 ms + 16 35 380.2 ms 575.1 ms + 32 67 491.5 ms 1124.4 ms + 64 131 1165.9 ms 2414.1 ms + 128 259 2633.7 ms 6120.7 ms ────────────────────────────────────────────────────────────────────────────────────────  Scenario B — 1 ms async sleep per branch ← parallelism verification @@ -82,55 +83,55 @@ Latency p50 (×200 sequential runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 5.09 ms 6.50 ms - 32 67 7.41 ms 10.2 ms - 64 131 14.5 ms 21.2 ms - 128 259 33.1 ms 53.8 ms + 16 35 4.00 ms 8.53 ms + 32 67 5.54 ms 10.9 ms + 64 131 8.58 ms 22.6 ms + 128 259 20.3 ms 59.4 ms Latency p95 (×200 sequential runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 5.76 ms 7.17 ms - 32 67 7.91 ms 11.9 ms - 64 131 19.0 ms 24.3 ms - 128 259 134.8 ms 130.6 ms + 16 35 4.40 ms 9.73 ms + 32 67 9.33 ms 13.2 ms + 64 131 9.75 ms 26.0 ms + 128 259 86.8 ms 128.3 ms Latency p99 (×200 sequential runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 21.9 ms 13.9 ms - 32 67 79.1 ms 13.8 ms - 64 131 107.7 ms 98.0 ms - 128 259 169.2 ms 134.5 ms + 16 35 4.61 ms 11.3 ms + 32 67 43.7 ms 13.9 ms + 64 131 79.0 ms 90.2 ms + 128 259 103.8 ms 131.6 ms Overhead per extra branch-pair (slope of p50 vs N, 2 branches per unit) Each N increment adds 2 async branches (one per phase) - Timbal +255.1 µs / branch-pair (+127.5 µs per branch) - Flow steps +430.9 µs / branch-pair (+215.4 µs per branch) + Timbal +147.4 µs / branch-pair (+73.7 µs per branch) + Flow steps +468.6 µs / branch-pair (+234.3 µs per branch) Burst p50 (100 concurrent runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 728.6 ms 612.8 ms - 32 67 1255.1 ms 1163.3 ms - 64 131 2498.7 ms 2548.8 ms - 128 259 5531.2 ms 6346.3 ms + 16 35 335.6 ms 777.1 ms + 32 67 622.3 ms 1196.1 ms + 64 131 1479.6 ms 2749.1 ms + 128 259 3339.8 ms 6925.6 ms Burst p95 (100 concurrent runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 730.7 ms 680.1 ms - 32 67 1259.0 ms 1246.6 ms - 64 131 2506.6 ms 2656.2 ms - 128 259 5629.3 ms 6595.8 ms + 16 35 338.3 ms 885.5 ms + 32 67 626.4 ms 1285.3 ms + 64 131 1486.6 ms 2861.0 ms + 128 259 3423.5 ms 7188.2 ms Burst p99 (100 concurrent runs) N steps Timbal Flow steps ────── ─────── ────────────── ────────────── - 16 35 731.0 ms 686.8 ms - 32 67 1259.5 ms 1254.3 ms - 64 131 2507.2 ms 2666.7 ms - 128 259 5631.0 ms 6621.9 ms + 16 35 338.6 ms 895.7 ms + 32 67 626.8 ms 1293.8 ms + 64 131 1487.2 ms 2871.4 ms + 128 259 3424.8 ms 7205.3 ms Each phase: theoretical serial = N×1000 µs, parallel ≈ 1000 µs flat Two phases: expect ~2000 µs floor + scheduling overhead @@ -143,18 +144,18 @@ Sequential peak (per-run, cleared between) N steps T (InMemory) T (no trace) Flow steps ────── ─────── ────────────── ────────────── ────────────── - 16 35 294.8 KB 283.4 KB 345.3 KB - 32 67 504.1 KB 499.3 KB 352.2 KB - 64 131 936.7 KB 936.2 KB 403.5 KB - 128 259 1.7 MB 1.7 MB 694.2 KB + 16 35 229.6 KB 210.2 KB 346.7 KB + 32 67 385.0 KB 383.0 KB 345.6 KB + 64 131 725.9 KB 724.5 KB 391.9 KB + 128 259 1.4 MB 1.4 MB 678.2 KB Burst peak (100 concurrent, no GC between) N steps T (InMemory) T (no trace) Flow steps ────── ─────── ────────────── ────────────── ────────────── - 16 35 23.2 MB 23.2 MB 11.2 MB - 32 67 43.6 MB 43.6 MB 17.9 MB - 64 131 84.4 MB 84.4 MB 31.7 MB - 128 259 165.7 MB 165.7 MB 58.8 MB + 16 35 14.7 MB 14.7 MB 11.0 MB + 32 67 27.2 MB 27.2 MB 17.7 MB + 64 131 52.3 MB 52.3 MB 31.4 MB + 128 259 102.8 MB 102.8 MB 58.3 MB ──────────────────────────────────────────────────────────────────────────────────────── Topology: root → [N×p1] → aggregator → [N×p2] → sink diff --git a/benchmarks/crewai/results/bench_parallel.txt b/benchmarks/crewai/results/bench_parallel.txt index 86f3f603..a0aa6030 100644 --- a/benchmarks/crewai/results/bench_parallel.txt +++ b/benchmarks/crewai/results/bench_parallel.txt @@ -1,3 +1,4 @@ +Installed 61 packages in 119ms ════════════════════════════════════════════════════════════════════════════════════════  Timbal vs CrewAI Flow — Wide Parallel Fan-out @@ -17,61 +18,61 @@ Latency p50 (×200 sequential runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 802.8 µs 2.12 ms - 8 1.16 ms 2.61 ms - 16 1.87 ms 3.44 ms - 32 3.45 ms 5.81 ms - 64 7.13 ms 10.0 ms + 4 465.8 µs 2.76 ms + 8 637.0 µs 2.93 ms + 16 936.5 µs 3.85 ms + 32 1.65 ms 5.99 ms + 64 3.01 ms 9.94 ms Latency p95 (×200 sequential runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 1.02 ms 2.74 ms - 8 1.40 ms 3.21 ms - 16 2.46 ms 4.07 ms - 32 4.10 ms 8.14 ms - 64 8.51 ms 11.8 ms + 4 726.5 µs 3.41 ms + 8 990.3 µs 3.45 ms + 16 1.33 ms 4.63 ms + 32 2.08 ms 7.18 ms + 64 3.63 ms 11.0 ms Latency p99 (×200 sequential runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 1.38 ms 3.75 ms - 8 1.72 ms 3.63 ms - 16 9.46 ms 4.31 ms - 32 4.44 ms 58.7 ms - 64 12.3 ms 42.7 ms + 4 847.3 µs 5.99 ms + 8 1.13 ms 3.81 ms + 16 1.57 ms 5.20 ms + 32 2.48 ms 8.26 ms + 64 3.86 ms 11.9 ms Overhead per extra branch (slope of p50 latency vs N) Estimated by linear regression over the N values above - Timbal +105.8 µs / branch - Flow steps +133.3 µs / branch + Timbal +42.5 µs / branch + Flow steps +122.8 µs / branch Burst p50 (200 concurrent runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 112.7 ms 245.2 ms - 8 256.1 ms 346.3 ms - 16 509.8 ms 566.6 ms - 32 940.2 ms 955.0 ms - 64 2470.9 ms 1847.2 ms + 4 54.8 ms 274.6 ms + 8 83.6 ms 344.0 ms + 16 223.6 ms 583.7 ms + 32 444.3 ms 929.8 ms + 64 837.0 ms 2120.1 ms Burst p95 (200 concurrent runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 115.0 ms 359.0 ms - 8 259.2 ms 483.8 ms - 16 513.2 ms 691.3 ms - 32 945.1 ms 1116.8 ms - 64 2479.4 ms 2018.6 ms + 4 57.7 ms 415.0 ms + 8 87.0 ms 474.5 ms + 16 227.9 ms 720.4 ms + 32 450.0 ms 1065.2 ms + 64 845.8 ms 2299.5 ms Burst p99 (200 concurrent runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 115.2 ms 368.8 ms - 8 259.4 ms 494.3 ms - 16 513.5 ms 703.6 ms - 32 945.5 ms 1133.7 ms - 64 2480.2 ms 2032.3 ms + 4 57.9 ms 426.9 ms + 8 87.3 ms 486.5 ms + 16 228.3 ms 733.0 ms + 32 450.5 ms 1078.3 ms + 64 846.5 ms 2317.2 ms ────────────────────────────────────────────────────────────────────────────────────────  Scenario B — 1 ms async sleep per branch ← parallelism verification @@ -80,61 +81,61 @@ Latency p50 (×200 sequential runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 1.96 ms 3.22 ms - 8 2.14 ms 3.37 ms - 16 2.60 ms 3.86 ms - 32 3.64 ms 5.39 ms - 64 6.82 ms 9.82 ms + 4 1.60 ms 3.47 ms + 8 1.73 ms 3.83 ms + 16 1.96 ms 4.46 ms + 32 2.68 ms 6.37 ms + 64 3.98 ms 11.7 ms Latency p95 (×200 sequential runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 2.26 ms 3.84 ms - 8 2.38 ms 4.15 ms - 16 2.90 ms 4.31 ms - 32 4.25 ms 6.08 ms - 64 7.66 ms 11.0 ms + 4 1.81 ms 4.15 ms + 8 2.04 ms 4.49 ms + 16 2.31 ms 5.25 ms + 32 3.27 ms 7.37 ms + 64 4.71 ms 13.8 ms Latency p99 (×200 sequential runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 3.04 ms 10.5 ms - 8 2.65 ms 4.65 ms - 16 13.8 ms 4.61 ms - 32 4.82 ms 6.42 ms - 64 79.8 ms 86.9 ms + 4 1.95 ms 4.48 ms + 8 2.27 ms 5.06 ms + 16 2.51 ms 5.79 ms + 32 3.92 ms 8.04 ms + 64 4.91 ms 76.9 ms Overhead per extra branch (slope of p50 latency vs N) Estimated by linear regression over the N values above - Timbal +81.4 µs / branch - Flow steps +111.8 µs / branch + Timbal +40.3 µs / branch + Flow steps +138.1 µs / branch Burst p50 (200 concurrent runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 206.5 ms 327.4 ms - 8 272.4 ms 510.3 ms - 16 626.5 ms 690.1 ms - 32 1184.1 ms 1194.6 ms - 64 2420.1 ms 2470.2 ms + 4 63.9 ms 350.5 ms + 8 175.7 ms 419.3 ms + 16 338.9 ms 715.1 ms + 32 618.2 ms 1219.0 ms + 64 1280.1 ms 2533.9 ms Burst p95 (200 concurrent runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 209.2 ms 432.9 ms - 8 276.1 ms 621.7 ms - 16 630.6 ms 806.5 ms - 32 1189.7 ms 1324.4 ms - 64 2429.6 ms 2644.1 ms + 4 66.8 ms 477.9 ms + 8 179.4 ms 557.1 ms + 16 343.0 ms 853.6 ms + 32 623.5 ms 1370.8 ms + 64 1289.1 ms 2716.3 ms Burst p99 (200 concurrent runs) N Timbal Flow steps ────── ────────────── ────────────── - 4 209.4 ms 442.7 ms - 8 276.4 ms 631.4 ms - 16 630.9 ms 817.0 ms - 32 1190.2 ms 1336.3 ms - 64 2430.3 ms 2659.5 ms + 4 67.0 ms 490.3 ms + 8 179.7 ms 568.9 ms + 16 343.3 ms 867.2 ms + 32 624.5 ms 1385.1 ms + 64 1289.7 ms 2731.9 ms Theoretical serial: N × 1000 µs | Theoretical parallel: ~1000 µs flat Both frameworks use asyncio.gather — expect near-flat latency across widths diff --git a/benchmarks/crewai/results/bench_workflow.txt b/benchmarks/crewai/results/bench_workflow.txt index b645125c..b2ba8f7c 100644 --- a/benchmarks/crewai/results/bench_workflow.txt +++ b/benchmarks/crewai/results/bench_workflow.txt @@ -1,3 +1,4 @@ +Installed 61 packages in 123ms ════════════════════════════════════════════════════════════════════════════════════════  Timbal Workflow vs CrewAI Flow — DAG benchmark @@ -22,36 +23,36 @@ Latency (×200) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - mean 1.30 ms 1.53 ms 1.93 ms - p50 1.26 ms 1.47 ms 1.91 ms - p95 1.56 ms 1.99 ms 2.19 ms - p99 1.80 ms 2.49 ms 2.36 ms + mean 381.3 µs 1.66 ms 2.61 ms + p50 351.0 µs 1.59 ms 2.44 ms + p95 573.9 µs 2.28 ms 3.42 ms + p99 667.3 µs 2.78 ms 4.72 ms Memory (×200 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 63.3 KB 324 B - Flow bare 333.8 KB 1709 B - Flow+OI 359.5 KB 1840 B + Timbal 41.6 KB 213 B + Flow bare 328.1 KB 1680 B + Flow+OI 357.4 KB 1830 B Burst (200 concurrent) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - p50 114.11 ms 142.41 ms 270.87 ms - p75 115.60 ms 183.87 ms 314.92 ms - p95 200.29 ms 218.72 ms 351.31 ms - p99 200.68 ms 226.83 ms 358.26 ms - max 201.21 ms 229.83 ms 359.42 ms + p50 31.54 ms 181.79 ms 283.99 ms + p75 34.54 ms 235.03 ms 338.80 ms + p95 36.62 ms 291.64 ms 382.56 ms + p99 37.16 ms 301.49 ms 390.00 ms + max 37.31 ms 304.64 ms 391.55 ms - wall: Timbal: 201.2 ms | Flow bare: 229.8 ms | Flow+OI: 359.4 ms + wall: Timbal: 37.3 ms | Flow bare: 304.6 ms | Flow+OI: 391.6 ms Throughput (500 runs) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - 1 699/s 627/s 532/s - 10 1203/s 776/s 665/s - 50 961/s 992/s 563/s - 200 947/s 827/s 532/s + 1 2094/s 492/s 468/s + 10 4037/s 602/s 617/s + 50 2897/s 656/s 615/s + 200 4491/s 754/s 540/s ────────────────────────────────────────────────────────────────────────────────────────  Scenario 2: Fan-out/in (A → [B, C, D] → E) @@ -63,36 +64,36 @@ Latency (×200) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - mean 1.42 ms 1.35 ms 1.89 ms - p50 1.42 ms 1.33 ms 1.87 ms - p95 1.66 ms 1.59 ms 2.17 ms - p99 1.98 ms 1.72 ms 2.35 ms + mean 497.6 µs 1.40 ms 1.98 ms + p50 455.0 µs 1.33 ms 1.94 ms + p95 763.5 µs 1.83 ms 2.59 ms + p99 891.1 µs 2.21 ms 2.99 ms Memory (×200 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 84.3 KB 431 B - Flow bare 368.2 KB 1885 B - Flow+OI 396.7 KB 2031 B + Timbal 57.2 KB 293 B + Flow bare 363.2 KB 1860 B + Flow+OI 389.4 KB 1994 B Burst (200 concurrent) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - p50 243.94 ms 128.24 ms 378.68 ms - p75 244.42 ms 168.77 ms 413.68 ms - p95 244.78 ms 201.08 ms 442.66 ms - p99 245.22 ms 207.77 ms 450.87 ms - max 245.24 ms 208.98 ms 452.08 ms + p50 48.26 ms 132.92 ms 281.92 ms + p75 49.70 ms 181.32 ms 321.95 ms + p95 50.58 ms 215.94 ms 355.23 ms + p99 50.78 ms 222.68 ms 362.40 ms + max 50.92 ms 224.10 ms 364.49 ms - wall: Timbal: 245.2 ms | Flow bare: 209.0 ms | Flow+OI: 452.1 ms + wall: Timbal: 50.9 ms | Flow bare: 224.1 ms | Flow+OI: 364.5 ms Throughput (500 runs) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - 1 696/s 693/s 458/s - 10 911/s 557/s 494/s - 50 727/s 737/s 468/s - 200 718/s 826/s 545/s + 1 2111/s 667/s 502/s + 10 3190/s 877/s 586/s + 50 2395/s 820/s 437/s + 200 1819/s 750/s 501/s ────────────────────────────────────────────────────────────────────────────────────────  Scenario 3: Diamond (A → [B, C] → D) @@ -104,36 +105,36 @@ Latency (×200) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - mean 1.23 ms 1.40 ms 1.80 ms - p50 1.23 ms 1.33 ms 1.79 ms - p95 1.41 ms 1.84 ms 2.16 ms - p99 1.64 ms 2.44 ms 2.43 ms + mean 403.2 µs 1.31 ms 1.59 ms + p50 372.8 µs 1.24 ms 1.55 ms + p95 636.1 µs 1.73 ms 1.88 ms + p99 814.6 µs 2.03 ms 2.38 ms Memory (×200 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 71.5 KB 366 B - Flow bare 336.8 KB 1724 B - Flow+OI 367.2 KB 1880 B + Timbal 50.3 KB 258 B + Flow bare 332.3 KB 1701 B + Flow+OI 358.8 KB 1837 B Burst (200 concurrent) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - p50 208.54 ms 114.87 ms 277.43 ms - p75 209.22 ms 151.22 ms 316.32 ms - p95 210.20 ms 180.84 ms 348.27 ms - p99 210.38 ms 188.42 ms 357.04 ms - max 210.39 ms 189.75 ms 358.76 ms + p50 40.61 ms 101.79 ms 301.83 ms + p75 42.09 ms 133.50 ms 369.16 ms + p95 43.25 ms 155.99 ms 491.04 ms + p99 43.52 ms 160.99 ms 497.21 ms + max 43.66 ms 162.72 ms 498.53 ms - wall: Timbal: 210.4 ms | Flow bare: 189.7 ms | Flow+OI: 358.8 ms + wall: Timbal: 43.7 ms | Flow bare: 162.7 ms | Flow+OI: 498.5 ms Throughput (500 runs) Timbal Flow bare Flow+OI ──────────── ────────────── ────────────── ────────────── - 1 795/s 757/s 577/s - 10 1235/s 848/s 721/s - 50 959/s 1031/s 606/s - 200 802/s 746/s 633/s + 1 2231/s 824/s 469/s + 10 3985/s 818/s 553/s + 50 2791/s 970/s 431/s + 200 2689/s 964/s 548/s ──────────────────────────────────────────────────────────────────────────────────────── Timbal: built-in InMemory tracing on every run — spans recorded per step. diff --git a/benchmarks/google_adk/results/bench_agent.txt b/benchmarks/google_adk/results/bench_agent.txt index ffe130ed..c30ffb7c 100644 --- a/benchmarks/google_adk/results/bench_agent.txt +++ b/benchmarks/google_adk/results/bench_agent.txt @@ -6,60 +6,60 @@ ------------------------------------------------------------------------------------------------ Latency - Timbal mean 589.7 us p50 571.9 us p95 764.5 us - ADK mean 1.51 ms p50 1.02 ms p95 1.34 ms + Timbal mean 226.3 us p50 222.8 us p95 230.5 us + ADK mean 2.56 ms p50 2.43 ms p95 3.23 ms Memory - Timbal 393 B/run - ADK 17,681 B/run + Timbal 308 B/run + ADK 21,026 B/run Burst - Timbal p50 16.63 ms wall 27.59 ms - ADK p50 25.88 ms wall 46.21 ms + Timbal p50 234.0 us wall 12.88 ms + ADK p50 92.07 ms wall 95.64 ms Throughput - c=1 Timbal 1626/s ADK 949/s - c=10 Timbal 1245/s ADK 1052/s - c=50 Timbal 1229/s ADK 1086/s + c=1 Timbal 4025/s ADK 351/s + c=10 Timbal 4044/s ADK 500/s + c=50 Timbal 3292/s ADK 492/s ------------------------------------------------------------------------------------------------  Scenario 2: 3-step chain ------------------------------------------------------------------------------------------------ Latency - Timbal mean 1.30 ms p50 1.27 ms p95 1.54 ms - ADK mean 3.50 ms p50 2.98 ms p95 3.48 ms + Timbal mean 503.3 us p50 476.9 us p95 663.0 us + ADK mean 5.83 ms p50 5.73 ms p95 7.58 ms Memory - Timbal 812 B/run - ADK 37,253 B/run + Timbal 521 B/run + ADK 43,064 B/run Burst - Timbal p50 27.76 ms wall 36.30 ms - ADK p50 67.93 ms wall 87.30 ms + Timbal p50 500.0 us wall 16.22 ms + ADK p50 123.88 ms wall 126.32 ms Throughput - c=1 Timbal 597/s ADK 320/s - c=10 Timbal 667/s ADK 317/s - c=50 Timbal 638/s ADK 309/s + c=1 Timbal 1973/s ADK 153/s + c=10 Timbal 1671/s ADK 213/s + c=50 Timbal 1931/s ADK 243/s ------------------------------------------------------------------------------------------------  Scenario 3: parallel tools ------------------------------------------------------------------------------------------------ Latency - Timbal mean 825.0 us p50 806.4 us p95 925.9 us - ADK mean 1.63 ms p50 1.60 ms p95 1.80 ms + Timbal mean 443.6 us p50 411.7 us p95 629.7 us + ADK mean 3.11 ms p50 2.97 ms p95 3.73 ms Memory - Timbal 632 B/run - ADK 23,814 B/run + Timbal 427 B/run + ADK 27,194 B/run Burst - Timbal p50 21.59 ms wall 33.90 ms - ADK p50 38.17 ms wall 62.65 ms + Timbal p50 9.89 ms wall 16.68 ms + ADK p50 94.22 ms wall 97.28 ms Throughput - c=1 Timbal 753/s ADK 521/s - c=10 Timbal 1190/s ADK 529/s - c=50 Timbal 871/s ADK 528/s + c=1 Timbal 2287/s ADK 365/s + c=10 Timbal 2664/s ADK 401/s + c=50 Timbal 2883/s ADK 478/s diff --git a/benchmarks/google_adk/results/bench_callbacks.txt b/benchmarks/google_adk/results/bench_callbacks.txt index 91e02c4e..04651295 100644 --- a/benchmarks/google_adk/results/bench_callbacks.txt +++ b/benchmarks/google_adk/results/bench_callbacks.txt @@ -3,18 +3,18 @@  Timbal hooks vs Google ADK callbacks benchmark ================================================================================================ Timbal: OK | ADK: OK - Timbal hooks latency... p50=763.2 us - Timbal hooks memory... 86 KB peak - Timbal hooks burst (40 concurrent)... p50=22.53 ms - Timbal hooks throughput conc=1... 1247/s - Timbal hooks throughput conc=10... 1521/s - Timbal hooks throughput conc=50... 1541/s - ADK callbacks latency... p50=1.01 ms - ADK callbacks memory... 1782 KB peak - ADK callbacks burst (40 concurrent)... p50=22.85 ms - ADK callbacks throughput conc=1... 929/s - ADK callbacks throughput conc=10... 1062/s - ADK callbacks throughput conc=50... 1054/s + Timbal hooks latency... p50=239.2 us + Timbal hooks memory... 45 KB peak + Timbal hooks burst (40 concurrent)... p50=239.2 us + Timbal hooks throughput conc=1... 2858/s + Timbal hooks throughput conc=10... 2660/s + Timbal hooks throughput conc=50... 2883/s + ADK callbacks latency... p50=2.22 ms + ADK callbacks memory... 2408 KB peak + ADK callbacks burst (40 concurrent)... p50=70.32 ms + ADK callbacks throughput conc=1... 399/s + ADK callbacks throughput conc=10... 517/s + ADK callbacks throughput conc=50... 526/s ------------------------------------------------------------------------------------------------  Callback-enabled single-tool loop @@ -23,32 +23,32 @@ Latency (x100 sequential runs) Timbal hooks ADK callbacks ------------ ------------------ ------------------ - mean 797.8 us 1.05 ms - p50 763.2 us 1.01 ms - p95 1.00 ms 1.34 ms - p99 1.60 ms 1.55 ms + mean 247.4 us 2.48 ms + p50 239.2 us 2.22 ms + p95 289.5 us 2.65 ms + p99 534.5 us 23.84 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal hooks 86.4 KB 884 B - ADK callbacks 1781.7 KB 18245 B + Timbal hooks 45.1 KB 462 B + ADK callbacks 2408.2 KB 24660 B Burst (40 concurrent) Timbal hooks ADK callbacks ------------ ------------------ ------------------ - p50 22.53 ms 22.85 ms - p75 22.85 ms 23.10 ms - p95 23.16 ms 23.32 ms - p99 23.22 ms 23.43 ms - max 23.22 ms 23.43 ms + p50 239.2 us 70.32 ms + p75 258.4 us 92.45 ms + p95 556.7 us 93.09 ms + p99 20.91 ms 93.39 ms + max 20.91 ms 93.39 ms Throughput (200 loops) Timbal hooks ADK callbacks ------------ ------------------ ------------------ - 1 1247/s 929/s - 10 1521/s 1062/s - 50 1541/s 1054/s + 1 2858/s 399/s + 10 2660/s 517/s + 50 2883/s 526/s ------------------------------------------------------------------------------------------------ Both columns run no-op lifecycle callbacks around model/tool execution. diff --git a/benchmarks/google_adk/results/bench_transfer.txt b/benchmarks/google_adk/results/bench_transfer.txt index 8987876d..a7acba09 100644 --- a/benchmarks/google_adk/results/bench_transfer.txt +++ b/benchmarks/google_adk/results/bench_transfer.txt @@ -4,18 +4,18 @@ 100 iters | 40 burst | 100 mem | 200 throughput ops ================================================================================================ Timbal: OK | ADK: OK - Timbal delegation latency... p50=696.3 us - Timbal delegation memory... 76 KB peak - Timbal delegation burst (40 concurrent)... p50=20.05 ms - Timbal delegation throughput conc=1... 1360/s - Timbal delegation throughput conc=10... 1378/s - Timbal delegation throughput conc=50... 1405/s - ADK transfer latency... p50=977.1 us - ADK transfer memory... 1800 KB peak - ADK transfer burst (40 concurrent)... p50=21.49 ms - ADK transfer throughput conc=1... 968/s - ADK transfer throughput conc=10... 1028/s - ADK transfer throughput conc=50... 1037/s + Timbal delegation latency... p50=343.0 us + Timbal delegation memory... 62 KB peak + Timbal delegation burst (40 concurrent)... p50=331.9 us + Timbal delegation throughput conc=1... 2028/s + Timbal delegation throughput conc=10... 2113/s + Timbal delegation throughput conc=50... 2147/s + ADK transfer latency... p50=2.52 ms + ADK transfer memory... 5582 KB peak + ADK transfer burst (40 concurrent)... p50=84.51 ms + ADK transfer throughput conc=1... 331/s + ADK transfer throughput conc=10... 437/s + ADK transfer throughput conc=50... 220/s ------------------------------------------------------------------------------------------------  Delegation / transfer @@ -24,32 +24,32 @@ Latency (x100 sequential runs) Timbal delegation ADK transfer ------------ ------------------ ------------------ - mean 722.4 us 1.01 ms - p50 696.3 us 977.1 us - p95 872.6 us 1.21 ms - p99 1.09 ms 1.64 ms + mean 367.5 us 2.94 ms + p50 343.0 us 2.52 ms + p95 562.5 us 3.47 ms + p99 692.1 us 24.05 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal delegation 76.3 KB 781 B - ADK transfer 1800.1 KB 18433 B + Timbal delegation 62.0 KB 635 B + ADK transfer 5581.6 KB 57156 B Burst (40 concurrent) Timbal delegation ADK transfer ------------ ------------------ ------------------ - p50 20.05 ms 21.49 ms - p75 20.24 ms 21.95 ms - p95 20.43 ms 22.40 ms - p99 20.67 ms 22.41 ms - max 20.67 ms 22.41 ms + p50 331.9 us 84.51 ms + p75 391.3 us 108.71 ms + p95 711.8 us 109.38 ms + p99 21.54 ms 109.68 ms + max 21.54 ms 109.68 ms Throughput (200 loops) Timbal delegation ADK transfer ------------ ------------------ ------------------ - 1 1360/s 968/s - 10 1378/s 1028/s - 50 1405/s 1037/s + 1 2028/s 331/s + 10 2113/s 437/s + 50 2147/s 220/s ------------------------------------------------------------------------------------------------ Google ADK has a true sub-agent transfer primitive via transfer_to_agent. diff --git a/benchmarks/langchain/bench_long_conversation.py b/benchmarks/langchain/bench_long_conversation.py new file mode 100644 index 00000000..289b46b0 --- /dev/null +++ b/benchmarks/langchain/bench_long_conversation.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +""" +Timbal vs LangGraph — long conversations / long messages benchmark. + +Measures how framework overhead scales with conversation length and message +size. LLMs are faked on both sides (immediate text answer, no tools), so the +numbers isolate message validation, history processing, memory persistence, +and tracing — the per-turn cost a long-running chat assistant pays on every +request. + +Scenarios: + 1. Long history, single turn (stateless): an N-message conversation is + passed into one agent turn. N scales. + 2. Long messages, single turn: 20-message history, message size scales. + 3. Multi-turn session (stateful): T turns on one conversation. Timbal + persists memory via parent_id chaining (in-memory tracing provider); + LangGraph uses its MemorySaver checkpointer. Both rebuild/extend the + conversation each turn, so per-turn latency grows with turn index. + +Timbal runs with built-in tracing on (always-on by design). LangGraph runs +bare, per request — this is Timbal-with-observability vs LangGraph-without. + +Run: + uv run python benchmarks/langchain/bench_long_conversation.py + uv run python benchmarks/langchain/bench_long_conversation.py --quick +""" + +from __future__ import annotations + +import argparse +import logging +import os +import warnings + +logging.disable(logging.WARNING) +os.environ.setdefault("TIMBAL_LOG_LEVEL", "CRITICAL") +warnings.filterwarnings("ignore") + +import structlog # noqa: E402 + +structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.CRITICAL)) + +import asyncio # noqa: E402 +import gc # noqa: E402 +import statistics # noqa: E402 +import time # noqa: E402 +import tracemalloc # noqa: E402 + +parser = argparse.ArgumentParser(add_help=False) +parser.add_argument("--quick", action="store_true") +_args, _ = parser.parse_known_args() + +N_ITERS = 10 if _args.quick else 50 +N_WARMUP = 3 if _args.quick else 8 +N_MEM = 10 if _args.quick else 50 +HISTORY_SIZES = [10, 50] if _args.quick else [10, 50, 200] +MESSAGE_SIZES = [200, 5_000] if _args.quick else [200, 5_000, 20_000] +SESSION_TURNS = 8 if _args.quick else 20 +N_SESSIONS = 5 if _args.quick else 15 +WIDTH = 76 + +# ── Display helpers ────────────────────────────────────────────────────────── + +RESET = "\033[0m" +BOLD = "\033[1m" +CYAN = "\033[36m" +DIM = "\033[2m" + + +def section(title: str) -> None: + print() + print(f"{BOLD}{CYAN}{'─' * WIDTH}{RESET}") + print(f"{BOLD}{CYAN} {title}{RESET}") + print(f"{BOLD}{CYAN}{'─' * WIDTH}{RESET}") + + +def subsection(title: str) -> None: + print(f"\n {BOLD}{title}{RESET}") + + +def fmt_us(us: float) -> str: + if us >= 1_000: + return f"{us / 1_000:>8.2f} ms" + return f"{us:>8.1f} µs" + + +def pct(samples: list[float], p: float) -> float: + idx = min(int(len(samples) * p / 100), len(samples) - 1) + return sorted(samples)[idx] + + +# ═══════════════════════════════════════════════════════════════════════════════ +# TIMBAL +# ═══════════════════════════════════════════════════════════════════════════════ + +from timbal import Agent # noqa: E402 +from timbal.core.test_model import TestModel # noqa: E402 +from timbal.state.tracing.providers.in_memory import InMemoryTracingProvider # noqa: E402 +from timbal.types.message import Message # noqa: E402 + + +def _clear_traces(): + InMemoryTracingProvider._storage.clear() + + +def _make_timbal_agent() -> Agent: + """Fake LLM: immediately answers with a short text (no tools).""" + return Agent( + name="chat_agent", + model=TestModel(handler=lambda messages: f"ok ({len(messages)} msgs seen)"), + tools=[], + ) + + +def _timbal_history(n_messages: int, text: str) -> list[Message]: + """Alternating user/assistant history ending in a user message.""" + history = [] + for i in range(n_messages - 1): + role = "user" if i % 2 == 0 else "assistant" + history.append(Message.validate({"role": role, "content": f"[{i}] {text}"})) + history.append(Message.validate({"role": "user", "content": f"[final] {text}"})) + return history + + +# ═══════════════════════════════════════════════════════════════════════════════ +# LANGGRAPH +# ═══════════════════════════════════════════════════════════════════════════════ + +from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel # noqa: E402 +from langchain_core.messages import AIMessage, HumanMessage # noqa: E402 +from langgraph.checkpoint.memory import MemorySaver # noqa: E402 +from langgraph.prebuilt import create_react_agent # noqa: E402 + + +class _FakeLLM(FakeMessagesListChatModel): + """Stateless fake LLM: always answers with a short text (no tool calls).""" + + def bind_tools(self, tools, **kw): + return self + + async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs): + from langchain_core.outputs import ChatGeneration, ChatResult + + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content=f"ok ({len(messages)} msgs seen)"))] + ) + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + from langchain_core.outputs import ChatGeneration, ChatResult + + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content=f"ok ({len(messages)} msgs seen)"))] + ) + + +def _make_lc_graph(checkpointer=None): + llm = _FakeLLM(responses=[AIMessage(content="unused")]) + return create_react_agent(llm, tools=[], checkpointer=checkpointer) + + +def _lc_history(n_messages: int, text: str) -> list: + history = [] + for i in range(n_messages - 1): + cls = HumanMessage if i % 2 == 0 else AIMessage + history.append(cls(content=f"[{i}] {text}")) + history.append(HumanMessage(content=f"[final] {text}")) + return history + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Measurement helpers +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def _latency(run_fn, n: int, warmup: int) -> list[float]: + for _ in range(warmup): + await run_fn() + _clear_traces() + gc.collect() + samples = [] + for _ in range(n): + t0 = time.perf_counter() + await run_fn() + samples.append((time.perf_counter() - t0) * 1e6) + # Clear per iteration (outside the timed window) so accumulated + # in-memory traces don't skew later samples — mirrors _memory. + _clear_traces() + return samples + + +async def _memory(run_fn, n: int, warmup: int) -> float: + for _ in range(warmup): + await run_fn() + _clear_traces() + gc.collect() + tracemalloc.start() + for _ in range(n): + await run_fn() + _clear_traces() + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + return peak / n + + +def _print_scaling_table(row_label: str, rows: list, t_p50s: list, lg_p50s: list) -> None: + print(f" {row_label:>12} {'Timbal p50':>12} {'LG p50':>12} {'ratio':>8}") + print(f" {'─' * 12} {'─' * 12} {'─' * 12} {'─' * 8}") + for row, t, lg in zip(rows, t_p50s, lg_p50s): + print(f" {row:>12} {fmt_us(t):>12} {fmt_us(lg):>12} {lg / t:>7.2f}x") + + +def _print_slope(unit: str, rows: list, t_p50s: list, lg_p50s: list, scale: float = 1.0) -> None: + dn = (rows[-1] - rows[0]) * scale + t_slope = (t_p50s[-1] - t_p50s[0]) / dn + lg_slope = (lg_p50s[-1] - lg_p50s[0]) / dn + subsection(f"Marginal cost per {unit} (slope {rows[0]} → {rows[-1]})") + print(f" Timbal +{t_slope:.2f} µs / {unit}") + print(f" LG (bare) +{lg_slope:.2f} µs / {unit}") + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Scenario 1 — Long history, single turn (stateless) +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def bench_long_history() -> None: + section(f"Scenario 1: Long history, single turn (N messages in → 1 LLM turn, ×{N_ITERS})") + print(f" {DIM}History injected per call (stateless). ~60-char messages.{RESET}") + + t_agent = _make_timbal_agent() + lg_graph = _make_lc_graph() + text = "x" * 40 + + t_p50s, lg_p50s = [], [] + for n in HISTORY_SIZES: + t_history = _timbal_history(n, text) + lg_history = _lc_history(n, text) + + async def t_run(h=t_history): + await t_agent(messages=h).collect() + + async def lg_run(h=lg_history): + await lg_graph.ainvoke({"messages": h}) + + t_p50s.append(pct(await _latency(t_run, N_ITERS, N_WARMUP), 50)) + lg_p50s.append(pct(await _latency(lg_run, N_ITERS, N_WARMUP), 50)) + + subsection("Latency p50 by history length") + _print_scaling_table("N messages", HISTORY_SIZES, t_p50s, lg_p50s) + _print_slope("message", HISTORY_SIZES, t_p50s, lg_p50s) + + # Memory at largest N + n = HISTORY_SIZES[-1] + t_history = _timbal_history(n, text) + lg_history = _lc_history(n, text) + + async def t_run_mem(): + await t_agent(messages=t_history).collect() + + async def lg_run_mem(): + await lg_graph.ainvoke({"messages": lg_history}) + + t_mem = await _memory(t_run_mem, N_MEM, N_WARMUP) + lg_mem = await _memory(lg_run_mem, N_MEM, N_WARMUP) + subsection(f"Memory per run (N={n}, ×{N_MEM} runs)") + print(f" Timbal {t_mem / 1024:>8.1f} KB") + print(f" LG (bare) {lg_mem / 1024:>8.1f} KB") + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Scenario 2 — Long messages, single turn +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def bench_long_messages() -> None: + n = 20 + section(f"Scenario 2: Long messages, single turn ({n}-message history, size scales, ×{N_ITERS})") + + t_agent = _make_timbal_agent() + lg_graph = _make_lc_graph() + + t_p50s, lg_p50s = [], [] + for size in MESSAGE_SIZES: + text = "lorem ipsum " * max(1, size // 12) + text = text[:size] + t_history = _timbal_history(n, text) + lg_history = _lc_history(n, text) + + async def t_run(h=t_history): + await t_agent(messages=h).collect() + + async def lg_run(h=lg_history): + await lg_graph.ainvoke({"messages": h}) + + t_p50s.append(pct(await _latency(t_run, N_ITERS, N_WARMUP), 50)) + lg_p50s.append(pct(await _latency(lg_run, N_ITERS, N_WARMUP), 50)) + + subsection("Latency p50 by message size") + _print_scaling_table("msg bytes", MESSAGE_SIZES, t_p50s, lg_p50s) + _print_slope("KB of message", MESSAGE_SIZES, t_p50s, lg_p50s, scale=1 / 1000) + + # Memory at largest size + size = MESSAGE_SIZES[-1] + text = ("lorem ipsum " * max(1, size // 12))[:size] + t_history = _timbal_history(n, text) + lg_history = _lc_history(n, text) + + async def t_run_mem(): + await t_agent(messages=t_history).collect() + + async def lg_run_mem(): + await lg_graph.ainvoke({"messages": lg_history}) + + t_mem = await _memory(t_run_mem, N_MEM, N_WARMUP) + lg_mem = await _memory(lg_run_mem, N_MEM, N_WARMUP) + subsection(f"Memory per run (msg={size} B, ×{N_MEM} runs)") + print(f" Timbal {t_mem / 1024:>8.1f} KB") + print(f" LG (bare) {lg_mem / 1024:>8.1f} KB") + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Scenario 3 — Multi-turn session (stateful memory) +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def _timbal_session(agent: Agent, turns: int, turn_samples: dict[int, list[float]] | None = None) -> str: + """Run one Timbal conversation: memory persists via parent_id chaining.""" + parent_id = None + last_text = "" + for t in range(1, turns + 1): + t0 = time.perf_counter() + kwargs = {"prompt": f"turn {t}"} + if parent_id is not None: + kwargs["parent_id"] = parent_id + out = await agent(**kwargs).collect() + elapsed = (time.perf_counter() - t0) * 1e6 + if turn_samples is not None and t in turn_samples: + turn_samples[t].append(elapsed) + parent_id = out.run_id + last_text = out.output.collect_text() + return last_text + + +async def _lc_session(graph, thread_id: str, turns: int, turn_samples: dict[int, list[float]] | None = None) -> str: + """Run one LangGraph conversation: memory persists via MemorySaver checkpointer.""" + config = {"configurable": {"thread_id": thread_id}} + last_text = "" + for t in range(1, turns + 1): + t0 = time.perf_counter() + result = await graph.ainvoke({"messages": [HumanMessage(content=f"turn {t}")]}, config) + elapsed = (time.perf_counter() - t0) * 1e6 + if turn_samples is not None and t in turn_samples: + turn_samples[t].append(elapsed) + last_text = result["messages"][-1].content + return last_text + + +async def bench_session() -> None: + section(f"Scenario 3: Multi-turn session ({SESSION_TURNS} turns, ×{N_SESSIONS} sessions)") + print(f" {DIM}Stateful memory: Timbal parent_id chaining (in-memory tracing provider){RESET}") + print(f" {DIM}vs LangGraph MemorySaver checkpointer. History grows every turn.{RESET}") + + t_agent = _make_timbal_agent() + checkpoints = sorted({1, max(2, SESSION_TURNS // 4), SESSION_TURNS // 2, SESSION_TURNS}) + + # Warmup one session each + await _timbal_session(t_agent, 3) + _clear_traces() + warm_graph = _make_lc_graph(checkpointer=MemorySaver()) + await _lc_session(warm_graph, "warmup", 3) + gc.collect() + + # Timed sessions + t_turns: dict[int, list[float]] = {c: [] for c in checkpoints} + t_walls: list[float] = [] + for _ in range(N_SESSIONS): + t0 = time.perf_counter() + await _timbal_session(t_agent, SESSION_TURNS, t_turns) + t_walls.append((time.perf_counter() - t0) * 1e3) + _clear_traces() + + lg_turns: dict[int, list[float]] = {c: [] for c in checkpoints} + lg_walls: list[float] = [] + for s in range(N_SESSIONS): + graph = _make_lc_graph(checkpointer=MemorySaver()) # fresh saver per session (≙ cleared traces) + t0 = time.perf_counter() + await _lc_session(graph, f"s{s}", SESSION_TURNS, lg_turns) + lg_walls.append((time.perf_counter() - t0) * 1e3) + + subsection("Per-turn latency p50 (history grows with turn index)") + print(f" {'turn':>12} {'Timbal':>12} {'LG (bare)':>12} {'ratio':>8}") + print(f" {'─' * 12} {'─' * 12} {'─' * 12} {'─' * 8}") + for c in checkpoints: + t = pct(t_turns[c], 50) + lg = pct(lg_turns[c], 50) + print(f" {c:>12} {fmt_us(t):>12} {fmt_us(lg):>12} {lg / t:>7.2f}x") + + subsection(f"Full-session wall time ({SESSION_TURNS} turns)") + print(f" Timbal {statistics.median(t_walls):>8.1f} ms") + print(f" LG (bare) {statistics.median(lg_walls):>8.1f} ms") + + # Memory per session + gc.collect() + tracemalloc.start() + for _ in range(3): + await _timbal_session(t_agent, SESSION_TURNS) + _clear_traces() + _, t_peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + gc.collect() + tracemalloc.start() + for s in range(3): + graph = _make_lc_graph(checkpointer=MemorySaver()) + await _lc_session(graph, f"m{s}", SESSION_TURNS) + _, lg_peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + subsection(f"Memory per session ({SESSION_TURNS} turns, peak / 3 sessions)") + print(f" Timbal {t_peak / 3 / 1024:>8.1f} KB") + print(f" LG (bare) {lg_peak / 3 / 1024:>8.1f} KB") + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Main +# ═══════════════════════════════════════════════════════════════════════════════ + + +async def _verify() -> None: + print(f"\n {DIM}Verifying correctness...{RESET}") + + # Stateless history: both fake LLMs must see the full injected history. + n = 10 + t_agent = _make_timbal_agent() + out = await (t_agent(messages=_timbal_history(n, "hello"))).collect() + t_text = out.output.collect_text() + _clear_traces() + + lg_graph = _make_lc_graph() + lg_out = await lg_graph.ainvoke({"messages": _lc_history(n, "hello")}) + lg_text = lg_out["messages"][-1].content + print(f" stateless n={n}: Timbal='{t_text}' LangGraph='{lg_text}'") + assert "10 msgs seen" in t_text, t_text + assert "10 msgs seen" in lg_text, lg_text + + # Stateful session: by turn 3 both must have accumulated 5 messages + # (u1, a1, u2, a2, u3) when the fake LLM is called. + t_text = await _timbal_session(_make_timbal_agent(), 3) + _clear_traces() + lg_text = await _lc_session(_make_lc_graph(checkpointer=MemorySaver()), "verify", 3) + print(f" session turn 3: Timbal='{t_text}' LangGraph='{lg_text}'") + assert "5 msgs seen" in t_text, t_text + assert "5 msgs seen" in lg_text, lg_text + print(f" {DIM}ok — both frameworks accumulate identical histories{RESET}") + + +async def main() -> None: + print() + print(f"{BOLD}{'═' * WIDTH}{RESET}") + print(f"{BOLD} Timbal vs LangGraph — long conversations / long messages{RESET}") + print(f" {N_ITERS} iters · histories {HISTORY_SIZES} · msg sizes {MESSAGE_SIZES}") + print(f" sessions: {N_SESSIONS} × {SESSION_TURNS} turns") + print(f" {DIM}LLMs faked (instant answer, no tools) — pure framework overhead.{RESET}") + print(f" {DIM}Timbal: built-in tracing on. LangGraph: bare (no LangSmith).{RESET}") + print(f"{BOLD}{'═' * WIDTH}{RESET}") + + await _verify() + await bench_long_history() + await bench_long_messages() + await bench_session() + + print() + print(f"{DIM}{'─' * WIDTH}") + print(f" Agent/graph creation excluded (except fresh per-session checkpointers,") + print(f" mirrored by Timbal's cleared traces). Histories prebuilt outside timing.") + print(f"{'─' * WIDTH}{RESET}") + print() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmarks/langchain/results/bench_agent.txt b/benchmarks/langchain/results/bench_agent.txt index 0d4acfbd..4e04590a 100644 --- a/benchmarks/langchain/results/bench_agent.txt +++ b/benchmarks/langchain/results/bench_agent.txt @@ -15,38 +15,38 @@ Latency (×100) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - mean 1.02 ms 2.71 ms 5.40 ms - p50 1.09 ms 2.64 ms 5.15 ms - p95 1.69 ms 3.45 ms 7.37 ms - p99 1.90 ms 3.83 ms 9.57 ms + mean 450.6 µs 3.66 ms 6.59 ms + p50 269.8 µs 3.64 ms 6.38 ms + p95 512.8 µs 4.60 ms 8.09 ms + p99 17.12 ms 5.16 ms 26.22 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 219.0 KB 2242 B - LG (bare) 286.1 KB 2930 B - LG + LangSmith 10774.7 KB 110333 B + Timbal 192.3 KB 1969 B + LG (bare) 397.6 KB 4072 B + LG + LangSmith 23054.2 KB 236075 B Burst (20 concurrent loops) Both use 1 shared instance. LangGraph uses native ainvoke(). - burst [ 1/20] last 7.67 ms burst [ 2/20] last 7.53 ms burst [ 3/20] last 7.53 ms burst [ 4/20] last 7.34 ms burst [ 5/20] last 7.27 ms burst [ 6/20] last 7.15 ms burst [ 7/20] last 7.12 ms burst [ 8/20] last 7.09 ms burst [ 9/20] last 7.08 ms burst [10/20] last 7.06 ms burst [11/20] last 6.71 ms burst [12/20] last 6.66 ms burst [13/20] last 6.96 ms burst [14/20] last 7.04 ms burst [15/20] last 7.04 ms burst [16/20] last 6.99 ms burst [17/20] last 6.98 ms burst [18/20] last 6.98 ms burst [19/20] last 6.97 ms burst [20/20] last 6.92 ms + burst [ 1/20] last 515.0 µs burst [ 2/20] last 352.5 µs burst [ 3/20] last 288.0 µs burst [ 4/20] last 281.8 µs burst [ 5/20] last 266.8 µs burst [ 6/20] last 257.0 µs burst [ 7/20] last 265.6 µs burst [ 8/20] last 229.7 µs burst [ 9/20] last 242.2 µs burst [10/20] last 489.6 µs burst [11/20] last 281.7 µs burst [12/20] last 246.1 µs burst [13/20] last 251.4 µs burst [14/20] last 240.9 µs burst [15/20] last 232.6 µs burst [16/20] last 235.2 µs burst [17/20] last 235.0 µs burst [18/20] last 227.2 µs burst [19/20] last 224.6 µs burst [20/20] last 16.39 ms Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - p50 7.06 ms 23.81 ms 59.00 ms - p75 7.27 ms 24.90 ms 60.25 ms - p95 7.67 ms 25.96 ms 61.97 ms - p99 7.67 ms 25.96 ms 61.97 ms - max 7.67 ms 25.96 ms 61.97 ms + p50 257.0 µs 27.46 ms 71.58 ms + p75 288.0 µs 45.96 ms 89.67 ms + p95 16.39 ms 47.30 ms 91.66 ms + p99 16.39 ms 47.30 ms 91.66 ms + max 16.39 ms 47.30 ms 91.66 ms - wall: Timbal 7.7 ms | LG bare 26.0 ms | LG+Smith 62.0 ms + wall: Timbal 16.4 ms | LG bare 47.3 ms | LG+Smith 91.7 ms Throughput (200 loops) Both use 1 shared instance, semaphore-bounded async. concurrency Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - 1 1583/s 340/s 167/s - 10 1716/s 662/s 224/s - 50 1810/s 654/s 226/s + 1 2698/s 313/s 147/s + 10 2545/s 529/s 211/s + 50 2267/s 640/s 254/s ────────────────────────────────────────────────────────────────────────────  Scenario 2: Multi-step (LLM → add → LLM → mul → LLM → sub → LLM → answer) @@ -55,38 +55,38 @@ Latency (×100) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - mean 1.12 ms 5.87 ms 14.98 ms - p50 1.10 ms 5.54 ms 13.57 ms - p95 1.93 ms 8.31 ms 19.49 ms - p99 2.54 ms 10.76 ms 73.49 ms + mean 514.1 µs 6.43 ms 13.16 ms + p50 305.5 µs 6.52 ms 12.84 ms + p95 827.0 µs 7.12 ms 16.16 ms + p99 17.26 ms 7.93 ms 33.98 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 409.9 KB 4198 B - LG (bare) 168.3 KB 1724 B - LG + LangSmith 22746.8 KB 232927 B + Timbal 92.9 KB 952 B + LG (bare) 405.4 KB 4151 B + LG + LangSmith 36974.8 KB 378622 B Burst (10 concurrent loops) Both use 1 shared instance. LangGraph uses native ainvoke(). - burst [ 1/10] last 11.91 ms burst [ 2/10] last 11.92 ms burst [ 3/10] last 12.27 ms burst [ 4/10] last 12.30 ms burst [ 5/10] last 12.32 ms burst [ 6/10] last 12.26 ms burst [ 7/10] last 12.68 ms burst [ 8/10] last 12.69 ms burst [ 9/10] last 12.48 ms burst [10/10] last 12.36 ms + burst [ 1/10] last 903.4 µs burst [ 2/10] last 705.5 µs burst [ 3/10] last 566.8 µs burst [ 4/10] last 521.8 µs burst [ 5/10] last 514.1 µs burst [ 6/10] last 691.9 µs burst [ 7/10] last 792.0 µs burst [ 8/10] last 533.1 µs burst [ 9/10] last 16.68 ms burst [10/10] last 526.0 µs Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - p50 12.32 ms 29.96 ms 80.76 ms - p75 12.48 ms 30.40 ms 81.07 ms - p95 12.69 ms 31.04 ms 81.54 ms - p99 12.69 ms 31.04 ms 81.54 ms - max 12.69 ms 31.04 ms 81.54 ms + p50 691.9 µs 54.45 ms 151.59 ms + p75 792.0 µs 55.00 ms 154.67 ms + p95 16.68 ms 55.74 ms 156.79 ms + p99 16.68 ms 55.74 ms 156.79 ms + max 16.68 ms 55.74 ms 156.79 ms - wall: Timbal 12.7 ms | LG bare 31.0 ms | LG+Smith 81.5 ms + wall: Timbal 16.7 ms | LG bare 55.7 ms | LG+Smith 156.8 ms Throughput (200 loops) Both use 1 shared instance, semaphore-bounded async. concurrency Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - 1 618/s 168/s 85/s - 10 750/s 276/s 101/s - 50 818/s 238/s 104/s + 1 1494/s 129/s 72/s + 10 1500/s 193/s 97/s + 50 1195/s 315/s 105/s ────────────────────────────────────────────────────────────────────────────  Scenario 3: Parallel tools (LLM → [add, mul, neg] → LLM → answer) @@ -95,38 +95,38 @@ Latency (×100) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - mean 1.03 ms 3.32 ms 8.06 ms - p50 1.04 ms 3.32 ms 6.85 ms - p95 1.76 ms 3.83 ms 11.93 ms - p99 2.51 ms 4.31 ms 58.92 ms + mean 528.2 µs 4.67 ms 8.88 ms + p50 328.2 µs 4.62 ms 8.54 ms + p95 569.4 µs 5.71 ms 11.03 ms + p99 17.96 ms 7.38 ms 25.89 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 322.3 KB 3301 B - LG (bare) 343.9 KB 3521 B - LG + LangSmith 14081.2 KB 144191 B + Timbal 272.0 KB 2786 B + LG (bare) 439.0 KB 4495 B + LG + LangSmith 26844.6 KB 274889 B Burst (15 concurrent loops) Both use 1 shared instance. LangGraph uses native ainvoke(). - burst [ 1/15] last 10.25 ms burst [ 2/15] last 10.23 ms burst [ 3/15] last 10.26 ms burst [ 4/15] last 10.29 ms burst [ 5/15] last 12.21 ms burst [ 6/15] last 12.17 ms burst [ 7/15] last 12.20 ms burst [ 8/15] last 12.24 ms burst [ 9/15] last 12.26 ms burst [10/15] last 12.29 ms burst [11/15] last 12.32 ms burst [12/15] last 12.50 ms burst [13/15] last 11.97 ms burst [14/15] last 11.73 ms burst [15/15] last 11.23 ms + burst [ 1/15] last 21.11 ms burst [ 2/15] last 21.02 ms burst [ 3/15] last 21.01 ms burst [ 4/15] last 21.01 ms burst [ 5/15] last 21.01 ms burst [ 6/15] last 21.02 ms burst [ 7/15] last 21.03 ms burst [ 8/15] last 21.03 ms burst [ 9/15] last 21.06 ms burst [10/15] last 21.08 ms burst [11/15] last 21.09 ms burst [12/15] last 21.11 ms burst [13/15] last 21.14 ms burst [14/15] last 4.08 ms burst [15/15] last 4.07 ms Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - p50 12.17 ms 28.63 ms 65.36 ms - p75 12.26 ms 29.33 ms 65.79 ms - p95 12.50 ms 30.21 ms 66.22 ms - p99 12.50 ms 30.21 ms 66.22 ms - max 12.50 ms 30.21 ms 66.22 ms + p50 21.03 ms 48.15 ms 270.97 ms + p75 21.09 ms 48.81 ms 272.30 ms + p95 21.14 ms 49.68 ms 274.29 ms + p99 21.14 ms 49.68 ms 274.29 ms + max 21.14 ms 49.68 ms 274.29 ms - wall: Timbal 12.5 ms | LG bare 30.2 ms | LG+Smith 66.2 ms + wall: Timbal 21.1 ms | LG bare 49.7 ms | LG+Smith 274.3 ms Throughput (200 loops) Both use 1 shared instance, semaphore-bounded async. concurrency Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - 1 1030/s 274/s 82/s - 10 990/s 236/s 106/s - 50 606/s 328/s 164/s + 1 1686/s 223/s 127/s + 10 1648/s 363/s 157/s + 50 1677/s 469/s 179/s ────────────────────────────────────────────────────────────────────────────  AgentExecutor vs create_react_agent vs Timbal (Scenario 2: 3-tool chain) diff --git a/benchmarks/langchain/results/bench_double_fanout.txt b/benchmarks/langchain/results/bench_double_fanout.txt index d84bbe02..e6cc19a1 100644 --- a/benchmarks/langchain/results/bench_double_fanout.txt +++ b/benchmarks/langchain/results/bench_double_fanout.txt @@ -23,56 +23,56 @@ Latency p50 (×200 sequential runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 3.91 ms 3.29 ms 7.53 ms - 32 67 7.36 ms 5.78 ms 13.4 ms - 64 131 15.8 ms 12.4 ms 25.4 ms - 128 259 37.2 ms 29.9 ms 58.3 ms + 16 35 2.67 ms 5.43 ms 11.6 ms + 32 67 3.84 ms 7.72 ms 16.0 ms + 64 131 7.41 ms 13.7 ms 26.5 ms + 128 259 17.6 ms 32.1 ms 57.8 ms Latency p95 (×200 sequential runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 4.88 ms 4.16 ms 9.37 ms - 32 67 8.10 ms 6.51 ms 18.2 ms - 64 131 17.7 ms 14.2 ms 30.5 ms - 128 259 47.9 ms 34.7 ms 308.5 ms + 16 35 3.56 ms 6.43 ms 30.2 ms + 32 67 4.55 ms 8.89 ms 24.5 ms + 64 131 8.64 ms 15.8 ms 32.9 ms + 128 259 19.9 ms 39.7 ms 175.8 ms Latency p99 (×200 sequential runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 31.4 ms 20.3 ms 54.6 ms - 32 67 9.12 ms 7.05 ms 127.2 ms - 64 131 224.1 ms 198.0 ms 368.1 ms - 128 259 775.4 ms 507.1 ms 1051.9 ms + 16 35 6.35 ms 7.27 ms 54.4 ms + 32 67 7.41 ms 45.9 ms 90.8 ms + 64 131 9.12 ms 108.1 ms 168.6 ms + 128 259 58.1 ms 323.1 ms 408.4 ms Overhead per extra branch-pair (slope of p50 vs N, 2 branches per unit) Each N increment adds 2 async branches (one per phase) - Timbal +300.9 µs / branch-pair (+150.4 µs per branch) - LG (bare) +241.3 µs / branch-pair (+120.6 µs per branch) - LG+Smith +456.1 µs / branch-pair (+228.0 µs per branch) + Timbal +136.3 µs / branch-pair (+68.2 µs per branch) + LG (bare) +242.5 µs / branch-pair (+121.2 µs per branch) + LG+Smith +418.5 µs / branch-pair (+209.3 µs per branch) Burst p50 (100 concurrent runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 488.1 ms 489.2 ms 968.2 ms - 32 67 1068.2 ms 1074.4 ms 1894.7 ms - 64 131 2675.1 ms 2427.6 ms 4187.6 ms - 128 259 5950.7 ms 5449.5 ms 9217.0 ms + 16 35 138.6 ms 329.7 ms 707.9 ms + 32 67 323.0 ms 632.5 ms 1356.1 ms + 64 131 624.4 ms 1314.7 ms 2686.1 ms + 128 259 1752.5 ms 3009.6 ms 5844.7 ms Burst p95 (100 concurrent runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 491.1 ms 499.3 ms 982.1 ms - 32 67 1073.0 ms 1085.6 ms 1909.6 ms - 64 131 2684.3 ms 2439.8 ms 4203.4 ms - 128 259 5965.0 ms 5464.8 ms 9237.4 ms + 16 35 168.1 ms 370.3 ms 754.0 ms + 32 67 325.1 ms 724.1 ms 1443.7 ms + 64 131 630.0 ms 1485.7 ms 2706.5 ms + 128 259 1761.5 ms 3029.6 ms 5871.3 ms Burst p99 (100 concurrent runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 491.5 ms 500.7 ms 983.9 ms - 32 67 1073.5 ms 1086.8 ms 1911.2 ms - 64 131 2685.1 ms 2441.2 ms 4205.2 ms - 128 259 5967.5 ms 5466.6 ms 9239.4 ms + 16 35 168.5 ms 371.9 ms 756.0 ms + 32 67 404.5 ms 726.3 ms 1446.0 ms + 64 131 786.0 ms 1487.3 ms 2865.5 ms + 128 259 2115.7 ms 3494.9 ms 6212.7 ms ────────────────────────────────────────────────────────────────────────────────────────  Scenario B — 1 ms async sleep per branch ← parallelism verification @@ -85,56 +85,56 @@ Latency p50 (×200 sequential runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 6.17 ms 6.17 ms 9.51 ms - 32 67 7.75 ms 7.87 ms 14.8 ms - 64 131 14.7 ms 13.5 ms 26.3 ms - 128 259 33.9 ms 30.7 ms 54.4 ms + 16 35 4.07 ms 6.81 ms 13.2 ms + 32 67 5.52 ms 10.0 ms 17.6 ms + 64 131 9.10 ms 17.2 ms 30.8 ms + 128 259 19.9 ms 35.2 ms 61.1 ms Latency p95 (×200 sequential runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 10.6 ms 6.92 ms 11.4 ms - 32 67 9.25 ms 9.11 ms 20.2 ms - 64 131 16.2 ms 14.7 ms 36.2 ms - 128 259 43.2 ms 46.3 ms 101.6 ms + 16 35 4.64 ms 7.77 ms 18.3 ms + 32 67 6.32 ms 11.0 ms 20.9 ms + 64 131 11.2 ms 18.9 ms 38.9 ms + 128 259 22.9 ms 36.7 ms 72.8 ms Latency p99 (×200 sequential runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 118.7 ms 7.87 ms 63.6 ms - 32 67 103.7 ms 11.6 ms 154.4 ms - 64 131 224.9 ms 204.3 ms 332.4 ms - 128 259 488.5 ms 481.5 ms 872.0 ms + 16 35 6.62 ms 8.11 ms 41.1 ms + 32 67 47.5 ms 52.2 ms 83.8 ms + 64 131 115.0 ms 114.9 ms 176.9 ms + 128 259 226.8 ms 240.4 ms 334.1 ms Overhead per extra branch-pair (slope of p50 vs N, 2 branches per unit) Each N increment adds 2 async branches (one per phase) - Timbal +254.7 µs / branch-pair (+127.3 µs per branch) - LG (bare) +224.5 µs / branch-pair (+112.3 µs per branch) - LG+Smith +403.4 µs / branch-pair (+201.7 µs per branch) + Timbal +143.5 µs / branch-pair (+71.7 µs per branch) + LG (bare) +255.8 µs / branch-pair (+127.9 µs per branch) + LG+Smith +435.5 µs / branch-pair (+217.7 µs per branch) Burst p50 (100 concurrent runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 719.5 ms 573.6 ms 976.9 ms - 32 67 1290.5 ms 1273.0 ms 1831.8 ms - 64 131 3474.7 ms 2747.0 ms 3874.8 ms - 128 259 7522.0 ms 6452.9 ms 9914.0 ms + 16 35 173.7 ms 352.9 ms 749.1 ms + 32 67 315.8 ms 676.8 ms 1770.8 ms + 64 131 680.5 ms 1400.9 ms 2786.9 ms + 128 259 1740.2 ms 3122.0 ms 5660.7 ms Burst p95 (100 concurrent runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 727.4 ms 583.8 ms 990.1 ms - 32 67 1294.7 ms 1282.4 ms 1845.6 ms - 64 131 3486.2 ms 2760.3 ms 3893.4 ms - 128 259 7535.1 ms 6468.9 ms 9935.5 ms + 16 35 210.8 ms 394.6 ms 795.8 ms + 32 67 320.4 ms 768.0 ms 1873.2 ms + 64 131 685.6 ms 1582.0 ms 2809.9 ms + 128 259 1755.9 ms 3141.9 ms 5686.5 ms Burst p99 (100 concurrent runs) N steps Timbal LG (bare) LG+Smith ────── ─────── ────────────── ────────────── ────────────── - 16 35 728.0 ms 585.0 ms 991.7 ms - 32 67 1295.1 ms 1283.6 ms 1847.1 ms - 64 131 3487.0 ms 2761.9 ms 3895.6 ms - 128 259 7536.6 ms 6470.6 ms 9937.5 ms + 16 35 211.4 ms 396.1 ms 797.9 ms + 32 67 397.4 ms 769.7 ms 1875.8 ms + 64 131 847.6 ms 1583.8 ms 2996.1 ms + 128 259 2101.1 ms 3472.1 ms 6019.9 ms Each phase: theoretical serial = N×1000 µs, parallel ≈ 1000 µs flat Two phases: expect ~2000 µs floor + scheduling overhead @@ -147,18 +147,18 @@ Sequential peak (per-run, cleared between) N steps T (InMemory) T (no trace) LG (bare) ────── ─────── ────────────── ────────────── ────────────── - 16 35 278.0 KB 275.5 KB 277.7 KB - 32 67 488.7 KB 488.5 KB 485.8 KB - 64 131 917.9 KB 911.7 KB 897.8 KB - 128 259 1.7 MB 1.7 MB 1.7 MB + 16 35 169.1 KB 157.7 KB 284.6 KB + 32 67 272.3 KB 282.9 KB 511.7 KB + 64 131 511.1 KB 520.9 KB 960.0 KB + 128 259 1006.8 KB 995.6 KB 1.8 MB Burst peak (100 concurrent, no GC between) N steps T (InMemory) T (no trace) LG (bare) ────── ─────── ────────────── ────────────── ────────────── - 16 35 22.8 MB 22.8 MB 21.9 MB - 32 67 42.6 MB 42.6 MB 41.8 MB - 64 131 82.8 MB 82.8 MB 81.1 MB - 128 259 162.6 MB 162.6 MB 161.1 MB + 16 35 13.4 MB 13.4 MB 24.1 MB + 32 67 24.8 MB 24.8 MB 45.6 MB + 64 131 47.7 MB 47.7 MB 88.3 MB + 128 259 93.6 MB 93.6 MB 175.0 MB ──────────────────────────────────────────────────────────────────────────────────────── Topology: root → [N parallel branches] → aggregator → [N parallel branches] → sink diff --git a/benchmarks/langchain/results/bench_long_conversation.txt b/benchmarks/langchain/results/bench_long_conversation.txt new file mode 100644 index 00000000..a9cdb2e5 --- /dev/null +++ b/benchmarks/langchain/results/bench_long_conversation.txt @@ -0,0 +1,80 @@ + +════════════════════════════════════════════════════════════════════════════ + Timbal vs LangGraph — long conversations / long messages + 50 iters · histories [10, 50, 200] · msg sizes [200, 5000, 20000] + sessions: 15 × 20 turns + LLMs faked (instant answer, no tools) — pure framework overhead. + Timbal: built-in tracing on. LangGraph: bare (no LangSmith). +════════════════════════════════════════════════════════════════════════════ + + Verifying correctness... + stateless n=10: Timbal='ok (10 msgs seen)' LangGraph='ok (10 msgs seen)' + session turn 3: Timbal='ok (5 msgs seen)' LangGraph='ok (5 msgs seen)' + ok — both frameworks accumulate identical histories + +──────────────────────────────────────────────────────────────────────────── + Scenario 1: Long history, single turn (N messages in → 1 LLM turn, ×50) +──────────────────────────────────────────────────────────────────────────── + History injected per call (stateless). ~60-char messages. + + Latency p50 by history length + N messages Timbal p50 LG p50 ratio + ──────────── ──────────── ──────────── ──────── + 10 121.8 µs 1.10 ms 9.06x + 50 180.5 µs 1.44 ms 8.01x + 200 409.3 µs 2.93 ms 7.15x + + Marginal cost per message (slope 10 → 200) + Timbal +1.51 µs / message + LG (bare) +9.60 µs / message + + Memory per run (N=200, ×50 runs) + Timbal 1.1 KB + LG (bare) 2.9 KB + +──────────────────────────────────────────────────────────────────────────── + Scenario 2: Long messages, single turn (20-message history, size scales, ×50) +──────────────────────────────────────────────────────────────────────────── + + Latency p50 by message size + msg bytes Timbal p50 LG p50 ratio + ──────────── ──────────── ──────────── ──────── + 200 138.5 µs 1.20 ms 8.67x + 5000 135.0 µs 1.35 ms 10.00x + 20000 144.0 µs 2.42 ms 16.82x + + Marginal cost per KB of message (slope 200 → 20000) + Timbal +0.28 µs / KB of message + LG (bare) +61.75 µs / KB of message + + Memory per run (msg=20000 B, ×50 runs) + Timbal 0.8 KB + LG (bare) 18.9 KB + +──────────────────────────────────────────────────────────────────────────── + Scenario 3: Multi-turn session (20 turns, ×15 sessions) +──────────────────────────────────────────────────────────────────────────── + Stateful memory: Timbal parent_id chaining (in-memory tracing provider) + vs LangGraph MemorySaver checkpointer. History grows every turn. + + Per-turn latency p50 (history grows with turn index) + turn Timbal LG (bare) ratio + ──────────── ──────────── ──────────── ──────── + 1 125.1 µs 1.36 ms 10.91x + 5 117.1 µs 1.36 ms 11.60x + 10 129.0 µs 1.50 ms 11.63x + 20 152.5 µs 1.84 ms 12.08x + + Full-session wall time (20 turns) + Timbal 2.7 ms + LG (bare) 31.1 ms + + Memory per session (20 turns, peak / 3 sessions) + Timbal 45.2 KB + LG (bare) 158.7 KB + +──────────────────────────────────────────────────────────────────────────── + Agent/graph creation excluded (except fresh per-session checkpointers, + mirrored by Timbal's cleared traces). Histories prebuilt outside timing. +──────────────────────────────────────────────────────────────────────────── + diff --git a/benchmarks/langchain/results/bench_parallel.txt b/benchmarks/langchain/results/bench_parallel.txt index f512f9ce..1820ac18 100644 --- a/benchmarks/langchain/results/bench_parallel.txt +++ b/benchmarks/langchain/results/bench_parallel.txt @@ -19,62 +19,62 @@ Latency p50 (×200 sequential runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 894.2 µs 1.00 ms 2.04 ms - 8 1.26 ms 1.32 ms 2.72 ms - 16 2.06 ms 1.88 ms 4.15 ms - 32 3.51 ms 2.95 ms 6.69 ms - 64 6.84 ms 6.13 ms 12.6 ms + 4 453.3 µs 1.19 ms 2.27 ms + 8 607.7 µs 1.44 ms 2.98 ms + 16 1.00 ms 2.02 ms 4.60 ms + 32 1.67 ms 3.53 ms 7.77 ms + 64 3.11 ms 6.43 ms 13.3 ms Latency p95 (×200 sequential runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 1.20 ms 1.31 ms 3.02 ms - 8 1.72 ms 1.97 ms 4.27 ms - 16 2.68 ms 2.47 ms 5.73 ms - 32 4.39 ms 3.55 ms 8.54 ms - 64 7.96 ms 6.90 ms 15.5 ms + 4 707.0 µs 1.61 ms 3.44 ms + 8 924.7 µs 2.06 ms 3.97 ms + 16 1.36 ms 2.59 ms 5.84 ms + 32 2.11 ms 4.28 ms 15.8 ms + 64 3.60 ms 7.31 ms 16.3 ms Latency p99 (×200 sequential runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 1.53 ms 1.47 ms 3.94 ms - 8 2.00 ms 22.1 ms 5.17 ms - 16 3.68 ms 2.63 ms 7.37 ms - 32 19.3 ms 3.89 ms 9.33 ms - 64 32.2 ms 177.5 ms 243.7 ms + 4 881.5 µs 1.70 ms 4.28 ms + 8 2.63 ms 2.19 ms 5.35 ms + 16 3.10 ms 2.63 ms 38.4 ms + 32 3.58 ms 4.63 ms 60.2 ms + 64 4.19 ms 91.9 ms 101.5 ms Overhead per extra branch (slope of p50 latency vs N) Estimated by linear regression over the N values above - Timbal +98.9 µs / branch - LG (bare) +84.9 µs / branch - LG+Smith +175.7 µs / branch + Timbal +44.4 µs / branch + LG (bare) +88.5 µs / branch + LG+Smith +184.8 µs / branch Burst p50 (200 concurrent runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 122.8 ms 165.3 ms 297.7 ms - 8 258.0 ms 330.5 ms 517.8 ms - 16 546.3 ms 684.4 ms 911.5 ms - 32 1157.4 ms 1374.4 ms 1974.6 ms - 64 2223.6 ms 2457.7 ms 3643.5 ms + 4 51.3 ms 133.0 ms 288.0 ms + 8 77.7 ms 194.2 ms 426.7 ms + 16 138.4 ms 357.9 ms 682.1 ms + 32 261.5 ms 631.4 ms 1187.3 ms + 64 499.4 ms 1297.7 ms 2515.2 ms Burst p95 (200 concurrent runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 125.6 ms 182.0 ms 322.4 ms - 8 262.4 ms 347.3 ms 543.7 ms - 16 550.7 ms 700.8 ms 938.8 ms - 32 1163.3 ms 1391.5 ms 2002.0 ms - 64 2233.9 ms 2477.4 ms 3673.5 ms + 4 70.2 ms 154.5 ms 322.9 ms + 8 103.6 ms 214.6 ms 456.9 ms + 16 141.9 ms 379.5 ms 713.9 ms + 32 263.6 ms 653.1 ms 1218.9 ms + 64 504.6 ms 1322.0 ms 2551.1 ms Burst p99 (200 concurrent runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 126.1 ms 183.5 ms 324.7 ms - 8 262.7 ms 348.8 ms 546.0 ms - 16 551.2 ms 702.3 ms 941.1 ms - 32 1164.5 ms 1393.3 ms 2004.7 ms - 64 2234.5 ms 2479.1 ms 3676.2 ms + 4 70.7 ms 171.8 ms 341.5 ms + 8 103.9 ms 238.9 ms 483.0 ms + 16 182.5 ms 417.6 ms 753.2 ms + 32 329.6 ms 717.8 ms 1286.0 ms + 64 619.9 ms 1443.1 ms 2670.6 ms ────────────────────────────────────────────────────────────────────────────────────────  Scenario B — 1 ms async sleep per branch ← parallelism verification @@ -83,62 +83,62 @@ Latency p50 (×200 sequential runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 2.05 ms 2.36 ms 3.45 ms - 8 2.23 ms 2.58 ms 3.94 ms - 16 2.73 ms 3.07 ms 5.37 ms - 32 3.98 ms 4.59 ms 8.22 ms - 64 7.16 ms 6.93 ms 14.5 ms + 4 1.68 ms 2.59 ms 3.82 ms + 8 1.84 ms 3.07 ms 4.81 ms + 16 2.03 ms 3.57 ms 6.41 ms + 32 2.63 ms 5.05 ms 9.10 ms + 64 4.19 ms 8.44 ms 15.6 ms Latency p95 (×200 sequential runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 3.01 ms 2.99 ms 5.11 ms - 8 2.58 ms 2.95 ms 5.02 ms - 16 3.21 ms 3.46 ms 8.51 ms - 32 5.05 ms 5.27 ms 9.78 ms - 64 8.51 ms 8.25 ms 17.4 ms + 4 1.94 ms 3.04 ms 4.71 ms + 8 2.23 ms 8.43 ms 5.75 ms + 16 2.36 ms 3.98 ms 32.8 ms + 32 3.13 ms 5.74 ms 11.3 ms + 64 4.66 ms 9.33 ms 18.1 ms Latency p99 (×200 sequential runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 4.15 ms 3.86 ms 7.16 ms - 8 2.82 ms 3.21 ms 5.59 ms - 16 4.13 ms 3.67 ms 46.2 ms - 32 5.51 ms 5.63 ms 38.0 ms - 64 15.4 ms 191.9 ms 263.3 ms + 4 2.37 ms 3.21 ms 6.02 ms + 8 3.13 ms 18.3 ms 6.56 ms + 16 4.14 ms 4.28 ms 101.8 ms + 32 6.33 ms 5.99 ms 61.0 ms + 64 5.48 ms 97.4 ms 126.6 ms Overhead per extra branch (slope of p50 latency vs N) Estimated by linear regression over the N values above - Timbal +86.2 µs / branch - LG (bare) +77.7 µs / branch - LG+Smith +185.9 µs / branch + Timbal +41.7 µs / branch + LG (bare) +96.6 µs / branch + LG+Smith +192.6 µs / branch Burst p50 (200 concurrent runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 178.9 ms 187.1 ms 312.1 ms - 8 347.3 ms 342.8 ms 550.5 ms - 16 699.0 ms 638.2 ms 993.0 ms - 32 1438.9 ms 1335.9 ms 1939.0 ms - 64 2755.4 ms 2602.2 ms 4240.3 ms + 4 54.8 ms 142.4 ms 273.6 ms + 8 92.1 ms 203.5 ms 589.6 ms + 16 207.8 ms 461.7 ms 779.0 ms + 32 307.6 ms 717.9 ms 1750.0 ms + 64 613.6 ms 1435.0 ms 2835.4 ms Burst p95 (200 concurrent runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 182.1 ms 204.5 ms 332.6 ms - 8 350.1 ms 359.6 ms 580.1 ms - 16 702.8 ms 656.9 ms 1019.0 ms - 32 1445.5 ms 1354.1 ms 1966.8 ms - 64 2765.2 ms 2623.3 ms 4268.8 ms + 4 72.3 ms 162.6 ms 299.2 ms + 8 121.6 ms 223.0 ms 619.6 ms + 16 215.7 ms 557.4 ms 807.1 ms + 32 310.4 ms 740.9 ms 1784.4 ms + 64 621.1 ms 1459.8 ms 2871.3 ms Burst p99 (200 concurrent runs) N Timbal LG (bare) LG+Smith ────── ────────────── ────────────── ────────────── - 4 182.3 ms 206.1 ms 334.8 ms - 8 350.2 ms 361.0 ms 583.9 ms - 16 703.1 ms 658.6 ms 1021.6 ms - 32 1446.5 ms 1355.8 ms 1969.1 ms - 64 2765.8 ms 2625.1 ms 4271.3 ms + 4 72.6 ms 180.1 ms 318.1 ms + 8 122.1 ms 248.1 ms 647.6 ms + 16 260.7 ms 617.0 ms 849.4 ms + 32 379.1 ms 811.8 ms 1855.0 ms + 64 740.8 ms 1584.2 ms 3208.2 ms Theoretical serial: N × 1000 µs | Theoretical parallel: ~1000 µs flat Both frameworks use asyncio tasks — expect near-flat latency across widths diff --git a/benchmarks/langchain/results/bench_workflow.txt b/benchmarks/langchain/results/bench_workflow.txt index 8ffc4ac5..a2897ba6 100644 --- a/benchmarks/langchain/results/bench_workflow.txt +++ b/benchmarks/langchain/results/bench_workflow.txt @@ -18,36 +18,36 @@ Latency (×200) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - mean 1.17 ms 1.97 ms 2.58 ms - p50 1.13 ms 1.54 ms 2.23 ms - p95 1.48 ms 3.08 ms 3.75 ms - p99 1.88 ms 15.87 ms 15.30 ms + mean 403.8 µs 1.47 ms 2.16 ms + p50 328.9 µs 1.42 ms 1.97 ms + p95 416.8 µs 1.89 ms 3.02 ms + p99 698.6 µs 1.98 ms 4.33 ms Memory (×200 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 160.6 KB 822 B - LG (bare) 73.7 KB 377 B - LG + LangSmith 7161.0 KB 36664 B + Timbal 333.9 KB 1709 B + LG (bare) 76.9 KB 394 B + LG + LangSmith 29085.2 KB 148916 B Burst (500 concurrent) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - p50 415.24 ms 330.57 ms 644.38 ms - p75 417.63 ms 347.03 ms 660.97 ms - p95 418.62 ms 364.51 ms 673.93 ms - p99 419.34 ms 367.65 ms 677.88 ms - max 419.42 ms 368.66 ms 678.49 ms + p50 66.08 ms 328.81 ms 608.99 ms + p75 70.53 ms 346.87 ms 637.31 ms + p95 74.13 ms 363.06 ms 664.80 ms + p99 75.07 ms 367.26 ms 672.69 ms + max 75.42 ms 375.03 ms 684.35 ms - wall: Timbal 419.4 ms | LG bare 368.7 ms | LG+Smith 678.5 ms + wall: Timbal 75.4 ms | LG bare 375.0 ms | LG+Smith 684.3 ms Throughput (1000 runs) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - 1 814/s 883/s 407/s - 10 1171/s 1308/s 621/s - 50 1010/s 1068/s 568/s - 200 841/s 960/s 492/s + 1 2433/s 574/s 415/s + 10 2906/s 932/s 457/s + 50 4440/s 1005/s 577/s + 200 2807/s 990/s 636/s ────────────────────────────────────────────────────────────────────────────  Scenario 2: Fan-out/in (A → [B, C, D] → E) @@ -56,36 +56,36 @@ Latency (×200) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - mean 1.28 ms 1.33 ms 2.49 ms - p50 1.20 ms 1.30 ms 2.43 ms - p95 1.69 ms 1.67 ms 3.39 ms - p99 2.79 ms 1.84 ms 3.97 ms + mean 774.7 µs 1.82 ms 3.05 ms + p50 675.1 µs 1.76 ms 2.92 ms + p95 960.5 µs 2.45 ms 3.91 ms + p99 1.43 ms 3.40 ms 4.72 ms Memory (×200 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 444.6 KB 2277 B - LG (bare) 107.1 KB 548 B - LG + LangSmith 8486.3 KB 43450 B + Timbal 249.7 KB 1278 B + LG (bare) 382.8 KB 1960 B + LG + LangSmith 30708.4 KB 157227 B Burst (500 concurrent) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - p50 705.05 ms 549.38 ms 991.77 ms - p75 707.53 ms 567.97 ms 1016.86 ms - p95 709.55 ms 584.97 ms 1042.57 ms - p99 709.99 ms 588.58 ms 1047.41 ms - max 710.07 ms 589.63 ms 1048.35 ms + p50 109.31 ms 437.19 ms 734.58 ms + p75 111.90 ms 462.92 ms 753.33 ms + p95 114.78 ms 477.16 ms 770.51 ms + p99 115.30 ms 486.57 ms 779.40 ms + max 115.58 ms 488.16 ms 813.24 ms - wall: Timbal 710.1 ms | LG bare 589.6 ms | LG+Smith 1048.4 ms + wall: Timbal 115.6 ms | LG bare 488.2 ms | LG+Smith 813.2 ms Throughput (1000 runs) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - 1 635/s 674/s 432/s - 10 862/s 1005/s 479/s - 50 691/s 660/s 394/s - 200 515/s 565/s 331/s + 1 1820/s 600/s 327/s + 10 2728/s 825/s 428/s + 50 2905/s 917/s 501/s + 200 2456/s 1004/s 554/s ────────────────────────────────────────────────────────────────────────────  Scenario 3: Diamond (A → [B, C] → D) @@ -94,36 +94,36 @@ Latency (×200) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - mean 1.20 ms 1.31 ms 2.24 ms - p50 1.12 ms 1.26 ms 2.16 ms - p95 1.62 ms 1.72 ms 3.31 ms - p99 2.91 ms 1.83 ms 3.88 ms + mean 529.2 µs 1.46 ms 2.66 ms + p50 398.5 µs 1.40 ms 2.51 ms + p95 715.7 µs 1.94 ms 3.54 ms + p99 4.76 ms 2.24 ms 4.71 ms Memory (×200 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 197.1 KB 1009 B - LG (bare) 95.2 KB 487 B - LG + LangSmith 7251.3 KB 37127 B + Timbal 247.7 KB 1268 B + LG (bare) 95.3 KB 488 B + LG + LangSmith 29077.6 KB 148877 B Burst (500 concurrent) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - p50 449.95 ms 413.70 ms 753.63 ms - p75 450.90 ms 428.79 ms 787.28 ms - p95 452.30 ms 443.95 ms 859.53 ms - p99 452.79 ms 447.69 ms 883.94 ms - max 453.40 ms 448.73 ms 884.90 ms + p50 96.23 ms 359.21 ms 678.37 ms + p75 101.69 ms 416.87 ms 716.51 ms + p95 104.44 ms 579.03 ms 747.09 ms + p99 104.78 ms 610.14 ms 760.64 ms + max 105.07 ms 616.62 ms 762.98 ms - wall: Timbal 453.4 ms | LG bare 448.7 ms | LG+Smith 884.9 ms + wall: Timbal 105.1 ms | LG bare 616.6 ms | LG+Smith 763.0 ms Throughput (1000 runs) Timbal LG (bare) LG+Smith ──────────── ──────────── ──────────── ──────────── - 1 654/s 820/s 481/s - 10 1177/s 1110/s 535/s - 50 890/s 924/s 501/s - 200 661/s 526/s 221/s + 1 1650/s 668/s 412/s + 10 3071/s 949/s 472/s + 50 2750/s 1110/s 604/s + 200 2938/s 925/s 650/s ──────────────────────────────────────────────────────────────────────────── Both frameworks execute identical DAGs with trivial handlers. diff --git a/benchmarks/openai_agents/results/bench_agent.txt b/benchmarks/openai_agents/results/bench_agent.txt index 279ec585..6a393e41 100644 --- a/benchmarks/openai_agents/results/bench_agent.txt +++ b/benchmarks/openai_agents/results/bench_agent.txt @@ -10,73 +10,73 @@ Timbal: OK | OAI: OK [Scenario 1] - Timbal latency... p50=747.8 us - Timbal memory... 57 KB peak - Timbal burst (50 concurrent)... p50=17.44 ms - Timbal burst memory (50 concurrent)... 1876 KB peak - Timbal throughput conc=1... 1502/s - Timbal throughput conc=10... 1710/s - Timbal throughput conc=50... 1816/s - OAI (bare) latency... p50=1.53 ms - OAI (bare) memory... 118 KB peak - OAI (bare) burst (50 concurrent)... p50=22.86 ms - OAI (bare) burst memory (50 concurrent)... 1810 KB peak - OAI (bare) throughput conc=1... 462/s - OAI (bare) throughput conc=10... 1474/s - OAI (bare) throughput conc=50... 1722/s + Timbal latency... p50=286.5 us + Timbal memory... 42 KB peak + Timbal burst (50 concurrent)... p50=234.6 us + Timbal burst memory (50 concurrent)... 771 KB peak + Timbal throughput conc=1... 2929/s + Timbal throughput conc=10... 2955/s + Timbal throughput conc=50... 2716/s + OAI (bare) latency... p50=1.59 ms + OAI (bare) memory... 88 KB peak + OAI (bare) burst (50 concurrent)... p50=29.84 ms + OAI (bare) burst memory (50 concurrent)... 1949 KB peak + OAI (bare) throughput conc=1... 592/s + OAI (bare) throughput conc=10... 1176/s + OAI (bare) throughput conc=50... 1334/s OAI + tracing latency... p50=1.71 ms - OAI + tracing memory... 153 KB peak - OAI + tracing burst (50 concurrent)... p50=26.72 ms - OAI + tracing burst memory (50 concurrent)... 1932 KB peak - OAI + tracing throughput conc=1... 573/s - OAI + tracing throughput conc=10... 1357/s - OAI + tracing throughput conc=50... 1650/s + OAI + tracing memory... 116 KB peak + OAI + tracing burst (50 concurrent)... p50=26.75 ms + OAI + tracing burst memory (50 concurrent)... 2086 KB peak + OAI + tracing throughput conc=1... 537/s + OAI + tracing throughput conc=10... 440/s + OAI + tracing throughput conc=50... 1340/s [Scenario 2] - Timbal latency... p50=716.0 us - Timbal memory... 98 KB peak - Timbal burst (30 concurrent)... p50=29.09 ms - Timbal burst memory (30 concurrent)... 2085 KB peak - Timbal throughput conc=1... 647/s - Timbal throughput conc=10... 760/s - Timbal throughput conc=50... 812/s - OAI (bare) latency... p50=3.33 ms - OAI (bare) memory... 176 KB peak - OAI (bare) burst (30 concurrent)... p50=32.72 ms - OAI (bare) burst memory (30 concurrent)... 1420 KB peak - OAI (bare) throughput conc=1... 297/s - OAI (bare) throughput conc=10... 748/s - OAI (bare) throughput conc=50... 749/s - OAI + tracing latency... p50=3.36 ms - OAI + tracing memory... 209 KB peak - OAI + tracing burst (30 concurrent)... p50=34.04 ms - OAI + tracing burst memory (30 concurrent)... 1565 KB peak - OAI + tracing throughput conc=1... 292/s - OAI + tracing throughput conc=10... 690/s - OAI + tracing throughput conc=50... 695/s + Timbal latency... p50=277.6 us + Timbal memory... 70 KB peak + Timbal burst (30 concurrent)... p50=516.2 us + Timbal burst memory (30 concurrent)... 1122 KB peak + Timbal throughput conc=1... 1662/s + Timbal throughput conc=10... 1670/s + Timbal throughput conc=50... 1660/s + OAI (bare) latency... p50=3.46 ms + OAI (bare) memory... 105 KB peak + OAI (bare) burst (30 concurrent)... p50=53.49 ms + OAI (bare) burst memory (30 concurrent)... 1504 KB peak + OAI (bare) throughput conc=1... 270/s + OAI (bare) throughput conc=10... 542/s + OAI (bare) throughput conc=50... 719/s + OAI + tracing latency... p50=3.83 ms + OAI + tracing memory... 126 KB peak + OAI + tracing burst (30 concurrent)... p50=53.53 ms + OAI + tracing burst memory (30 concurrent)... 1672 KB peak + OAI + tracing throughput conc=1... 252/s + OAI + tracing throughput conc=10... 575/s + OAI + tracing throughput conc=50... 686/s [Scenario 3] - Timbal latency... p50=771.2 us - Timbal memory... 95 KB peak - Timbal burst (40 concurrent)... p50=21.58 ms - Timbal burst memory (40 concurrent)... 2531 KB peak - Timbal throughput conc=1... 974/s - Timbal throughput conc=10... 1118/s - Timbal throughput conc=50... 1144/s - OAI (bare) latency... p50=1.87 ms - OAI (bare) memory... 149 KB peak - OAI (bare) burst (40 concurrent)... p50=28.08 ms - OAI (bare) burst memory (40 concurrent)... 2066 KB peak - OAI (bare) throughput conc=1... 532/s - OAI (bare) throughput conc=10... 1125/s - OAI (bare) throughput conc=50... 1037/s - OAI + tracing latency... p50=1.83 ms - OAI + tracing memory... 177 KB peak - OAI + tracing burst (40 concurrent)... p50=28.96 ms - OAI + tracing burst memory (40 concurrent)... 2159 KB peak - OAI + tracing throughput conc=1... 515/s - OAI + tracing throughput conc=10... 1078/s - OAI + tracing throughput conc=50... 932/s + Timbal latency... p50=288.1 us + Timbal memory... 60 KB peak + Timbal burst (40 concurrent)... p50=9.14 ms + Timbal burst memory (40 concurrent)... 1292 KB peak + Timbal throughput conc=1... 1795/s + Timbal throughput conc=10... 2078/s + Timbal throughput conc=50... 2141/s + OAI (bare) latency... p50=1.93 ms + OAI (bare) memory... 105 KB peak + OAI (bare) burst (40 concurrent)... p50=29.58 ms + OAI (bare) burst memory (40 concurrent)... 2269 KB peak + OAI (bare) throughput conc=1... 488/s + OAI (bare) throughput conc=10... 904/s + OAI (bare) throughput conc=50... 1008/s + OAI + tracing latency... p50=2.04 ms + OAI + tracing memory... 127 KB peak + OAI + tracing burst (40 concurrent)... p50=32.69 ms + OAI + tracing burst memory (40 concurrent)... 2381 KB peak + OAI + tracing throughput conc=1... 429/s + OAI + tracing throughput conc=10... 865/s + OAI + tracing throughput conc=50... 997/s ------------------------------------------------------------------------------------------------  Scenario 1: Single tool (LLM -> add -> LLM -> answer) @@ -85,42 +85,42 @@ Latency (x100 sequential runs) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - mean 761.0 us 1.58 ms 1.73 ms - p50 747.8 us 1.53 ms 1.71 ms - p95 1.27 ms 2.04 ms 2.19 ms - p99 1.47 ms 2.32 ms 2.38 ms + mean 435.8 us 1.68 ms 1.97 ms + p50 286.5 us 1.59 ms 1.71 ms + p95 446.4 us 2.20 ms 2.39 ms + p99 16.08 ms 2.46 ms 18.30 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal 56.6 KB 579 B - OAI (bare) 117.6 KB 1204 B - OAI + tracing 153.3 KB 1570 B + Timbal 42.3 KB 433 B + OAI (bare) 87.6 KB 897 B + OAI + tracing 116.5 KB 1193 B Burst (50 concurrent) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - p50 17.44 ms 22.86 ms 26.72 ms - p75 17.56 ms 23.05 ms 26.84 ms - p95 17.84 ms 23.34 ms 27.17 ms - p99 18.35 ms 23.53 ms 27.41 ms - max 18.35 ms 23.53 ms 27.41 ms + p50 234.6 us 29.84 ms 26.75 ms + p75 240.2 us 47.77 ms 43.30 ms + p95 444.6 us 48.05 ms 43.52 ms + p99 16.45 ms 48.25 ms 43.83 ms + max 16.45 ms 48.25 ms 43.83 ms - wall: Timbal: 18.4 ms | OAI (bare): 23.5 ms | OAI + tracing: 27.4 ms + wall: Timbal: 16.5 ms | OAI (bare): 48.3 ms | OAI + tracing: 43.8 ms Burst memory (50 concurrent) framework peak per run ------------------------ ------------ ------------ - Timbal 1876.3 KB 38427 B - OAI (bare) 1809.5 KB 37059 B - OAI + tracing 1931.6 KB 39560 B + Timbal 770.7 KB 15785 B + OAI (bare) 1948.8 KB 39912 B + OAI + tracing 2086.4 KB 42729 B Throughput (200 loops) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - 1 1502/s 462/s 573/s - 10 1710/s 1474/s 1357/s - 50 1816/s 1722/s 1650/s + 1 2929/s 592/s 537/s + 10 2955/s 1176/s 440/s + 50 2716/s 1334/s 1340/s ------------------------------------------------------------------------------------------------  Scenario 2: Multi-step (LLM -> add -> LLM -> mul -> LLM -> sub -> LLM -> answer) @@ -129,42 +129,42 @@ Latency (x100 sequential runs) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - mean 765.8 us 3.36 ms 3.37 ms - p50 716.0 us 3.33 ms 3.36 ms - p95 1.15 ms 3.92 ms 3.90 ms - p99 2.59 ms 5.25 ms 4.30 ms + mean 433.5 us 3.62 ms 4.09 ms + p50 277.6 us 3.46 ms 3.83 ms + p95 381.8 us 4.35 ms 5.20 ms + p99 15.77 ms 5.26 ms 19.54 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal 98.5 KB 1009 B - OAI (bare) 176.3 KB 1806 B - OAI + tracing 208.7 KB 2137 B + Timbal 69.9 KB 716 B + OAI (bare) 104.6 KB 1071 B + OAI + tracing 126.3 KB 1294 B Burst (30 concurrent) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - p50 29.09 ms 32.72 ms 34.04 ms - p75 30.00 ms 32.89 ms 34.21 ms - p95 30.31 ms 33.29 ms 34.36 ms - p99 30.34 ms 33.43 ms 34.50 ms - max 30.34 ms 33.43 ms 34.50 ms + p50 516.2 us 53.49 ms 53.53 ms + p75 530.4 us 53.77 ms 53.56 ms + p95 868.0 us 54.06 ms 53.65 ms + p99 17.23 ms 54.29 ms 53.91 ms + max 17.23 ms 54.29 ms 53.91 ms - wall: Timbal: 30.3 ms | OAI (bare): 33.4 ms | OAI + tracing: 34.5 ms + wall: Timbal: 17.2 ms | OAI (bare): 54.3 ms | OAI + tracing: 53.9 ms Burst memory (30 concurrent) framework peak per run ------------------------ ------------ ------------ - Timbal 2085.0 KB 71169 B - OAI (bare) 1419.9 KB 48467 B - OAI + tracing 1564.8 KB 53411 B + Timbal 1122.3 KB 38307 B + OAI (bare) 1504.4 KB 51350 B + OAI + tracing 1672.0 KB 57072 B Throughput (200 loops) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - 1 647/s 297/s 292/s - 10 760/s 748/s 690/s - 50 812/s 749/s 695/s + 1 1662/s 270/s 252/s + 10 1670/s 542/s 575/s + 50 1660/s 719/s 686/s ------------------------------------------------------------------------------------------------  Scenario 3: Parallel tools (LLM -> [add, mul, neg] concurrent -> LLM -> answer) @@ -173,42 +173,42 @@ Latency (x100 sequential runs) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - mean 790.9 us 1.88 ms 1.90 ms - p50 771.2 us 1.87 ms 1.83 ms - p95 1.28 ms 2.39 ms 2.38 ms - p99 1.48 ms 2.61 ms 2.54 ms + mean 462.6 us 2.05 ms 2.33 ms + p50 288.1 us 1.93 ms 2.04 ms + p95 495.5 us 2.56 ms 2.93 ms + p99 16.75 ms 3.27 ms 18.18 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal 94.8 KB 971 B - OAI (bare) 149.2 KB 1528 B - OAI + tracing 176.7 KB 1809 B + Timbal 60.5 KB 619 B + OAI (bare) 104.6 KB 1071 B + OAI + tracing 126.6 KB 1296 B Burst (40 concurrent) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - p50 21.58 ms 28.08 ms 28.96 ms - p75 22.02 ms 28.54 ms 29.14 ms - p95 23.70 ms 28.79 ms 29.39 ms - p99 23.74 ms 28.95 ms 29.59 ms - max 23.74 ms 28.95 ms 29.59 ms + p50 9.14 ms 29.58 ms 32.69 ms + p75 25.79 ms 45.50 ms 48.36 ms + p95 25.93 ms 45.52 ms 48.46 ms + p99 26.08 ms 45.68 ms 48.73 ms + max 26.08 ms 45.68 ms 48.73 ms - wall: Timbal: 23.7 ms | OAI (bare): 28.9 ms | OAI + tracing: 29.6 ms + wall: Timbal: 26.1 ms | OAI (bare): 45.7 ms | OAI + tracing: 48.7 ms Burst memory (40 concurrent) framework peak per run ------------------------ ------------ ------------ - Timbal 2530.9 KB 64790 B - OAI (bare) 2066.0 KB 52890 B - OAI + tracing 2159.0 KB 55271 B + Timbal 1291.8 KB 33070 B + OAI (bare) 2268.6 KB 58076 B + OAI + tracing 2380.9 KB 60951 B Throughput (200 loops) Timbal OAI (bare) OAI + tracing ------------ ---------------- ---------------- ---------------- - 1 974/s 532/s 515/s - 10 1118/s 1125/s 1078/s - 50 1144/s 1037/s 932/s + 1 1795/s 488/s 429/s + 10 2078/s 904/s 865/s + 50 2141/s 1008/s 997/s ------------------------------------------------------------------------------------------------ Timbal: TestModel, built-in InMemory tracing always on. diff --git a/benchmarks/openai_agents/results/bench_agent_as_tool.txt b/benchmarks/openai_agents/results/bench_agent_as_tool.txt index 71333b34..6b590881 100644 --- a/benchmarks/openai_agents/results/bench_agent_as_tool.txt +++ b/benchmarks/openai_agents/results/bench_agent_as_tool.txt @@ -3,18 +3,18 @@  Timbal delegation vs OpenAI Agents SDK agent-as-tool benchmark ================================================================================================ Timbal: OK | OAI: OK - Timbal delegation latency... p50=726.3 us - Timbal delegation memory... 73 KB peak - Timbal delegation burst (40 concurrent)... p50=20.40 ms - Timbal delegation throughput conc=1... 1210/s - Timbal delegation throughput conc=10... 1314/s - Timbal delegation throughput conc=50... 844/s - OAI agent tool latency... p50=2.35 ms - OAI agent tool memory... 166 KB peak - OAI agent tool burst (40 concurrent)... p50=32.85 ms - OAI agent tool throughput conc=1... 440/s - OAI agent tool throughput conc=10... 964/s - OAI agent tool throughput conc=50... 870/s + Timbal delegation latency... p50=270.5 us + Timbal delegation memory... 61 KB peak + Timbal delegation burst (40 concurrent)... p50=349.2 us + Timbal delegation throughput conc=1... 2071/s + Timbal delegation throughput conc=10... 2202/s + Timbal delegation throughput conc=50... 2233/s + OAI agent tool latency... p50=2.56 ms + OAI agent tool memory... 124 KB peak + OAI agent tool burst (40 concurrent)... p50=37.98 ms + OAI agent tool throughput conc=1... 365/s + OAI agent tool throughput conc=10... 637/s + OAI agent tool throughput conc=50... 821/s ------------------------------------------------------------------------------------------------  Delegation / handoff @@ -23,32 +23,32 @@ Latency (x100 sequential runs) Timbal delegation OAI agent tool ------------ ---------------- ---------------- - mean 749.2 us 2.42 ms - p50 726.3 us 2.35 ms - p95 1.35 ms 2.90 ms - p99 1.48 ms 3.35 ms + mean 456.8 us 2.87 ms + p50 270.5 us 2.56 ms + p95 520.9 us 3.36 ms + p99 17.72 ms 20.25 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal delegation 73.2 KB 749 B - OAI agent tool 166.2 KB 1702 B + Timbal delegation 61.3 KB 627 B + OAI agent tool 123.8 KB 1268 B Burst (40 concurrent) Timbal delegation OAI agent tool ------------ ---------------- ---------------- - p50 20.40 ms 32.85 ms - p75 20.52 ms 33.13 ms - p95 20.64 ms 33.38 ms - p99 20.74 ms 33.56 ms - max 20.74 ms 33.56 ms + p50 349.2 us 37.98 ms + p75 378.5 us 54.95 ms + p95 698.6 us 55.10 ms + p99 17.30 ms 55.38 ms + max 17.30 ms 55.38 ms Throughput (200 loops) Timbal delegation OAI agent tool ------------ ---------------- ---------------- - 1 1210/s 440/s - 10 1314/s 964/s - 50 844/s 870/s + 1 2071/s 365/s + 10 2202/s 637/s + 50 2233/s 821/s ------------------------------------------------------------------------------------------------ Both columns use supervisor -> worker-as-tool -> supervisor final answer. diff --git a/benchmarks/openai_agents/results/bench_handoff.txt b/benchmarks/openai_agents/results/bench_handoff.txt index e9f43720..2e30ea50 100644 --- a/benchmarks/openai_agents/results/bench_handoff.txt +++ b/benchmarks/openai_agents/results/bench_handoff.txt @@ -4,18 +4,18 @@ 100 iters | 40 burst | 100 mem | 200 throughput ops ================================================================================================ Timbal: OK | OAI: OK - Timbal delegation latency... p50=730.6 us - Timbal delegation memory... 73 KB peak - Timbal delegation burst (40 concurrent)... p50=20.62 ms - Timbal delegation throughput conc=1... 1240/s - Timbal delegation throughput conc=10... 1388/s - Timbal delegation throughput conc=50... 1314/s - OAI handoff latency... p50=1.42 ms - OAI handoff memory... 120 KB peak - OAI handoff burst (40 concurrent)... p50=18.66 ms - OAI handoff throughput conc=1... 741/s - OAI handoff throughput conc=10... 1340/s - OAI handoff throughput conc=50... 1596/s + Timbal delegation latency... p50=273.1 us + Timbal delegation memory... 62 KB peak + Timbal delegation burst (40 concurrent)... p50=345.8 us + Timbal delegation throughput conc=1... 2299/s + Timbal delegation throughput conc=10... 2193/s + Timbal delegation throughput conc=50... 2205/s + OAI handoff latency... p50=1.46 ms + OAI handoff memory... 97 KB peak + OAI handoff burst (40 concurrent)... p50=20.35 ms + OAI handoff throughput conc=1... 604/s + OAI handoff throughput conc=10... 1006/s + OAI handoff throughput conc=50... 1296/s ------------------------------------------------------------------------------------------------  Delegation / handoff @@ -24,32 +24,32 @@ Latency (x100 sequential runs) Timbal delegation OAI handoff ------------ ---------------- ---------------- - mean 727.6 us 1.47 ms - p50 730.6 us 1.42 ms - p95 1.24 ms 1.95 ms - p99 1.42 ms 2.08 ms + mean 446.9 us 1.71 ms + p50 273.1 us 1.46 ms + p95 489.7 us 2.12 ms + p99 17.05 ms 18.66 ms Memory (x100 runs) framework peak per run ------------------------ ------------ ------------ - Timbal delegation 73.2 KB 749 B - OAI handoff 120.4 KB 1233 B + Timbal delegation 62.2 KB 637 B + OAI handoff 97.3 KB 996 B Burst (40 concurrent) Timbal delegation OAI handoff ------------ ---------------- ---------------- - p50 20.62 ms 18.66 ms - p75 21.09 ms 19.18 ms - p95 21.18 ms 19.58 ms - p99 21.36 ms 19.81 ms - max 21.36 ms 19.81 ms + p50 345.8 us 20.35 ms + p75 370.0 us 36.44 ms + p95 660.3 us 37.01 ms + p99 16.98 ms 37.33 ms + max 16.98 ms 37.33 ms Throughput (200 loops) Timbal delegation OAI handoff ------------ ---------------- ---------------- - 1 1240/s 741/s - 10 1388/s 1340/s - 50 1314/s 1596/s + 1 2299/s 604/s + 10 2193/s 1006/s + 50 2205/s 1296/s ------------------------------------------------------------------------------------------------ OpenAI Agents SDK has a true handoff primitive. diff --git a/benchmarks/pydantic/results/bench_agent.txt b/benchmarks/pydantic/results/bench_agent.txt index 787ca9eb..0f2fb15a 100644 --- a/benchmarks/pydantic/results/bench_agent.txt +++ b/benchmarks/pydantic/results/bench_agent.txt @@ -11,82 +11,82 @@ Timbal: ✓ | PAI: ✓ [Phase 1 · Scenario 1] - Timbal latency... p50=711.5 µs - Timbal memory... 58 KB peak - Timbal burst (50 concurrent)... p50=15.63 ms - Timbal burst memory (50 concurrent)... 1889 KB peak - Timbal throughput conc=1... 1492/s - Timbal throughput conc=10... 1649/s - Timbal throughput conc=50... 1814/s - PAI bare latency... p50=2.64 ms - PAI bare memory... 519 KB peak - PAI bare burst (50 concurrent)... p50=55.83 ms - PAI bare burst memory (50 concurrent)... 3169 KB peak - PAI bare throughput conc=1... 366/s - PAI bare throughput conc=10... 609/s - PAI bare throughput conc=50... 618/s + Timbal latency... p50=278.6 µs + Timbal memory... 42 KB peak + Timbal burst (50 concurrent)... p50=230.6 µs + Timbal burst memory (50 concurrent)... 768 KB peak + Timbal throughput conc=1... 2865/s + Timbal throughput conc=10... 2693/s + Timbal throughput conc=50... 2604/s + PAI bare latency... p50=3.05 ms + PAI bare memory... 3681 KB peak + PAI bare burst (50 concurrent)... p50=213.24 ms + PAI bare burst memory (50 concurrent)... 5507 KB peak + PAI bare throughput conc=1... 277/s + PAI bare throughput conc=10... 361/s + PAI bare throughput conc=50... 433/s [Phase 1 · Scenario 2] - Timbal latency... p50=783.7 µs - Timbal memory... 102 KB peak - Timbal burst (30 concurrent)... p50=29.80 ms - Timbal burst memory (30 concurrent)... 2042 KB peak - Timbal throughput conc=1... 654/s - Timbal throughput conc=10... 740/s - Timbal throughput conc=50... 822/s - PAI bare latency... p50=4.70 ms - PAI bare memory... 694 KB peak - PAI bare burst (30 concurrent)... p50=63.43 ms - PAI bare burst memory (30 concurrent)... 2074 KB peak - PAI bare throughput conc=1... 217/s - PAI bare throughput conc=10... 354/s - PAI bare throughput conc=50... 366/s + Timbal latency... p50=285.4 µs + Timbal memory... 70 KB peak + Timbal burst (30 concurrent)... p50=491.7 µs + Timbal burst memory (30 concurrent)... 1123 KB peak + Timbal throughput conc=1... 1572/s + Timbal throughput conc=10... 1514/s + Timbal throughput conc=50... 1556/s + PAI bare latency... p50=6.87 ms + PAI bare memory... 5353 KB peak + PAI bare burst (30 concurrent)... p50=115.26 ms + PAI bare burst memory (30 concurrent)... 3386 KB peak + PAI bare throughput conc=1... 162/s + PAI bare throughput conc=10... 227/s + PAI bare throughput conc=50... 248/s [Phase 1 · Scenario 3] - Timbal latency... p50=773.6 µs - Timbal memory... 91 KB peak - Timbal burst (40 concurrent)... p50=20.15 ms - Timbal burst memory (40 concurrent)... 2537 KB peak - Timbal throughput conc=1... 1007/s - Timbal throughput conc=10... 1108/s - Timbal throughput conc=50... 1154/s - PAI bare latency... p50=2.94 ms - PAI bare memory... 633 KB peak - PAI bare burst (40 concurrent)... p50=52.21 ms - PAI bare burst memory (40 concurrent)... 3416 KB peak - PAI bare throughput conc=1... 338/s - PAI bare throughput conc=10... 508/s - PAI bare throughput conc=50... 535/s + Timbal latency... p50=304.9 µs + Timbal memory... 61 KB peak + Timbal burst (40 concurrent)... p50=9.44 ms + Timbal burst memory (40 concurrent)... 1292 KB peak + Timbal throughput conc=1... 1666/s + Timbal throughput conc=10... 1827/s + Timbal throughput conc=50... 2021/s + PAI bare latency... p50=3.85 ms + PAI bare memory... 4087 KB peak + PAI bare burst (40 concurrent)... p50=84.71 ms + PAI bare burst memory (40 concurrent)... 5717 KB peak + PAI bare throughput conc=1... 165/s + PAI bare throughput conc=10... 336/s + PAI bare throughput conc=50... 368/s Activating Logfire instrumentation (send_to_logfire=False)... Logfire instrumented. [Phase 2 · Scenario 1] - PAI+Logfire latency... p50=3.67 ms - PAI+Logfire memory... 536 KB peak - PAI+Logfire burst (50 concurrent)... p50=78.28 ms - PAI+Logfire burst memory (50 concurrent)... 3730 KB peak - PAI+Logfire throughput conc=1... 278/s - PAI+Logfire throughput conc=10... 375/s - PAI+Logfire throughput conc=50... 408/s + PAI+Logfire latency... p50=5.35 ms + PAI+Logfire memory... 3935 KB peak + PAI+Logfire burst (50 concurrent)... p50=146.92 ms + PAI+Logfire burst memory (50 concurrent)... 6840 KB peak + PAI+Logfire throughput conc=1... 189/s + PAI+Logfire throughput conc=10... 239/s + PAI+Logfire throughput conc=50... 299/s [Phase 2 · Scenario 2] - PAI+Logfire latency... p50=6.70 ms - PAI+Logfire memory... 714 KB peak - PAI+Logfire burst (30 concurrent)... p50=112.51 ms - PAI+Logfire burst memory (30 concurrent)... 2487 KB peak - PAI+Logfire throughput conc=1... 164/s - PAI+Logfire throughput conc=10... 245/s - PAI+Logfire throughput conc=50... 243/s + PAI+Logfire latency... p50=10.17 ms + PAI+Logfire memory... 5761 KB peak + PAI+Logfire burst (30 concurrent)... p50=169.49 ms + PAI+Logfire burst memory (30 concurrent)... 4437 KB peak + PAI+Logfire throughput conc=1... 99/s + PAI+Logfire throughput conc=10... 125/s + PAI+Logfire throughput conc=50... 162/s [Phase 2 · Scenario 3] - PAI+Logfire latency... p50=3.85 ms - PAI+Logfire memory... 647 KB peak - PAI+Logfire burst (40 concurrent)... p50=152.77 ms - PAI+Logfire burst memory (40 concurrent)... 4286 KB peak - PAI+Logfire throughput conc=1... 255/s - PAI+Logfire throughput conc=10... 364/s - PAI+Logfire throughput conc=50... 361/s + PAI+Logfire latency... p50=6.41 ms + PAI+Logfire memory... 4432 KB peak + PAI+Logfire burst (40 concurrent)... p50=135.35 ms + PAI+Logfire burst memory (40 concurrent)... 7634 KB peak + PAI+Logfire throughput conc=1... 151/s + PAI+Logfire throughput conc=10... 202/s + PAI+Logfire throughput conc=50... 235/s ──────────────────────────────────────────────────────────────────────────────────────────  Scenario 1: Single tool (LLM → add → LLM → answer) @@ -95,42 +95,42 @@ Latency (×100 sequential runs) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - mean 741.7 µs 2.68 ms 3.76 ms - p50 711.5 µs 2.64 ms 3.67 ms - p95 1.25 ms 3.29 ms 4.70 ms - p99 1.42 ms 3.63 ms 5.06 ms + mean 491.9 µs 3.48 ms 5.61 ms + p50 278.6 µs 3.05 ms 5.35 ms + p95 383.4 µs 4.43 ms 6.58 ms + p99 22.51 ms 26.47 ms 29.47 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 58.4 KB 598 B - PAI (bare) 519.0 KB 5315 B - PAI+Logfire 535.9 KB 5488 B + Timbal 42.3 KB 434 B + PAI (bare) 3680.9 KB 37692 B + PAI+Logfire 3934.7 KB 40291 B Burst (50 concurrent — fully async on both frameworks) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - p50 15.63 ms 55.83 ms 78.28 ms - p75 16.66 ms 59.13 ms 81.26 ms - p95 16.81 ms 62.44 ms 84.89 ms - p99 16.81 ms 64.14 ms 88.06 ms - max 16.81 ms 64.14 ms 88.06 ms + p50 230.6 µs 213.24 ms 146.92 ms + p75 237.4 µs 217.03 ms 152.26 ms + p95 400.0 µs 247.03 ms 171.79 ms + p99 21.83 ms 253.56 ms 176.37 ms + max 21.83 ms 253.56 ms 176.37 ms - wall: Timbal: 16.8 ms | PAI (bare): 64.1 ms | PAI+Logfire: 88.1 ms + wall: Timbal: 21.8 ms | PAI (bare): 253.6 ms | PAI+Logfire: 176.4 ms Burst memory (50 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 1889.4 KB 38695 B - PAI (bare) 3169.1 KB 64903 B - PAI+Logfire 3730.3 KB 76396 B + Timbal 767.7 KB 15723 B + PAI (bare) 5507.1 KB 112786 B + PAI+Logfire 6840.1 KB 140086 B Throughput (200 loops) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - 1 1492/s 366/s 278/s - 10 1649/s 609/s 375/s - 50 1814/s 618/s 408/s + 1 2865/s 277/s 189/s + 10 2693/s 361/s 239/s + 50 2604/s 433/s 299/s ──────────────────────────────────────────────────────────────────────────────────────────  Scenario 2: Multi-step (LLM → add → LLM → mul → LLM → sub → LLM → answer) @@ -139,42 +139,42 @@ Latency (×100 sequential runs) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - mean 801.5 µs 4.69 ms 6.70 ms - p50 783.7 µs 4.70 ms 6.70 ms - p95 1.38 ms 5.48 ms 7.63 ms - p99 2.16 ms 5.63 ms 9.66 ms + mean 536.5 µs 7.08 ms 11.06 ms + p50 285.4 µs 6.87 ms 10.17 ms + p95 556.0 µs 7.79 ms 14.46 ms + p99 23.88 ms 31.01 ms 34.52 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 102.0 KB 1045 B - PAI (bare) 693.9 KB 7105 B - PAI+Logfire 713.6 KB 7307 B + Timbal 69.9 KB 716 B + PAI (bare) 5353.2 KB 54817 B + PAI+Logfire 5760.5 KB 58988 B Burst (30 concurrent — fully async on both frameworks) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - p50 29.80 ms 63.43 ms 112.51 ms - p75 30.02 ms 64.92 ms 115.93 ms - p95 30.98 ms 67.54 ms 118.00 ms - p99 31.00 ms 68.51 ms 119.53 ms - max 31.00 ms 68.51 ms 119.53 ms + p50 491.7 µs 115.26 ms 169.49 ms + p75 525.5 µs 140.98 ms 186.87 ms + p95 805.9 µs 143.72 ms 195.00 ms + p99 24.64 ms 145.58 ms 197.45 ms + max 24.64 ms 145.58 ms 197.45 ms - wall: Timbal: 31.0 ms | PAI (bare): 68.5 ms | PAI+Logfire: 119.5 ms + wall: Timbal: 24.6 ms | PAI (bare): 145.6 ms | PAI+Logfire: 197.4 ms Burst memory (30 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 2042.4 KB 69712 B - PAI (bare) 2073.7 KB 70784 B - PAI+Logfire 2486.6 KB 84874 B + Timbal 1123.1 KB 38337 B + PAI (bare) 3386.0 KB 115575 B + PAI+Logfire 4436.9 KB 151447 B Throughput (200 loops) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - 1 654/s 217/s 164/s - 10 740/s 354/s 245/s - 50 822/s 366/s 243/s + 1 1572/s 162/s 99/s + 10 1514/s 227/s 125/s + 50 1556/s 248/s 162/s ──────────────────────────────────────────────────────────────────────────────────────────  Scenario 3: Parallel tools (LLM → [add, mul, neg] concurrent → LLM → answer) @@ -183,42 +183,42 @@ Latency (×100 sequential runs) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - mean 784.4 µs 2.99 ms 3.97 ms - p50 773.6 µs 2.94 ms 3.85 ms - p95 1.42 ms 3.67 ms 4.93 ms - p99 1.64 ms 4.64 ms 5.14 ms + mean 561.2 µs 4.12 ms 6.72 ms + p50 304.9 µs 3.85 ms 6.41 ms + p95 574.0 µs 4.50 ms 7.88 ms + p99 23.59 ms 26.77 ms 33.04 ms Memory (×100 runs) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 91.1 KB 933 B - PAI (bare) 632.9 KB 6481 B - PAI+Logfire 646.7 KB 6622 B + Timbal 60.7 KB 621 B + PAI (bare) 4087.0 KB 41851 B + PAI+Logfire 4432.3 KB 45387 B Burst (40 concurrent — fully async on both frameworks) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - p50 20.15 ms 52.21 ms 152.77 ms - p75 22.59 ms 54.79 ms 163.79 ms - p95 22.76 ms 58.19 ms 173.27 ms - p99 22.79 ms 58.52 ms 173.42 ms - max 22.79 ms 58.52 ms 173.42 ms + p50 9.44 ms 84.71 ms 135.35 ms + p75 32.96 ms 106.34 ms 156.87 ms + p95 33.36 ms 114.82 ms 167.15 ms + p99 33.60 ms 115.99 ms 172.26 ms + max 33.60 ms 115.99 ms 172.26 ms - wall: Timbal: 22.8 ms | PAI (bare): 58.5 ms | PAI+Logfire: 173.4 ms + wall: Timbal: 33.6 ms | PAI (bare): 116.0 ms | PAI+Logfire: 172.3 ms Burst memory (40 concurrent — peak during full burst, no GC between) framework peak per run ──────────────────────── ──────────── ──────────── - Timbal 2536.6 KB 64937 B - PAI (bare) 3415.6 KB 87440 B - PAI+Logfire 4285.6 KB 109711 B + Timbal 1291.9 KB 33073 B + PAI (bare) 5716.9 KB 146353 B + PAI+Logfire 7633.8 KB 195426 B Throughput (200 loops) Timbal PAI (bare) PAI+Logfire ──────────── ────────────── ────────────── ────────────── - 1 1007/s 338/s 255/s - 10 1108/s 508/s 364/s - 50 1154/s 535/s 361/s + 1 1666/s 165/s 151/s + 10 1827/s 336/s 202/s + 50 2021/s 368/s 235/s ────────────────────────────────────────────────────────────────────────────────────────── Timbal: TestModel (sync handler, stateless). Built-in InMemory tracing always on. diff --git a/benchmarks/pydantic/results/bench_control_flow.txt b/benchmarks/pydantic/results/bench_control_flow.txt index eefff933..e28b4e14 100644 --- a/benchmarks/pydantic/results/bench_control_flow.txt +++ b/benchmarks/pydantic/results/bench_control_flow.txt @@ -10,106 +10,106 @@ Timbal: ✓ | PG: ✓ [Phase 1 · rounds=8] - Timbal latency... p50=5.22 ms - Timbal memory... 182.3 KB peak - Timbal burst (100 concurrent)... p50=337.3 ms - Timbal throughput conc=1... 175/s - Timbal throughput conc=10... 284/s - Timbal throughput conc=50... 273/s - Timbal throughput conc=200... 245/s - PG bare latency... p50=146.9 µs - PG bare memory... 16.4 KB peak - PG bare burst (100 concurrent)... p50=148.8 µs - PG bare throughput conc=1... 6414/s - PG bare throughput conc=10... 6526/s - PG bare throughput conc=50... 6596/s - PG bare throughput conc=200... 6473/s + Timbal latency... p50=3.41 ms + Timbal memory... 162.5 KB peak + Timbal burst (100 concurrent)... p50=137.0 ms + Timbal throughput conc=1... 257/s + Timbal throughput conc=10... 656/s + Timbal throughput conc=50... 768/s + Timbal throughput conc=200... 717/s + PG bare latency... p50=137.9 µs + PG bare memory... 17.8 KB peak + PG bare burst (100 concurrent)... p50=141.0 µs + PG bare throughput conc=1... 6025/s + PG bare throughput conc=10... 5962/s + PG bare throughput conc=50... 6016/s + PG bare throughput conc=200... 6051/s [Phase 1 · rounds=16] - Timbal latency... p50=11.3 ms - Timbal memory... 336.8 KB peak - Timbal burst (100 concurrent)... p50=836.4 ms - Timbal throughput conc=1... 81/s - Timbal throughput conc=10... 127/s - Timbal throughput conc=50... 125/s - Timbal throughput conc=200... 105/s - PG bare latency... p50=285.7 µs - PG bare memory... 15.8 KB peak - PG bare burst (100 concurrent)... p50=283.2 µs - PG bare throughput conc=1... 3389/s - PG bare throughput conc=10... 2279/s - PG bare throughput conc=50... 3411/s - PG bare throughput conc=200... 3417/s + Timbal latency... p50=5.75 ms + Timbal memory... 293.6 KB peak + Timbal burst (100 concurrent)... p50=307.0 ms + Timbal throughput conc=1... 153/s + Timbal throughput conc=10... 291/s + Timbal throughput conc=50... 324/s + Timbal throughput conc=200... 302/s + PG bare latency... p50=268.4 µs + PG bare memory... 16.6 KB peak + PG bare burst (100 concurrent)... p50=276.0 µs + PG bare throughput conc=1... 3201/s + PG bare throughput conc=10... 3200/s + PG bare throughput conc=50... 3172/s + PG bare throughput conc=200... 3241/s [Phase 1 · rounds=32] - Timbal latency... p50=23.8 ms - Timbal memory... 635.5 KB peak - Timbal burst (100 concurrent)... p50=1719.9 ms - Timbal throughput conc=1... 38/s - Timbal throughput conc=10... 57/s - Timbal throughput conc=50... 56/s - Timbal throughput conc=200... 48/s - PG bare latency... p50=559.2 µs - PG bare memory... 15.9 KB peak - PG bare burst (100 concurrent)... p50=565.4 µs - PG bare throughput conc=1... 1750/s - PG bare throughput conc=10... 1765/s - PG bare throughput conc=50... 1738/s - PG bare throughput conc=200... 1750/s + Timbal latency... p50=15.4 ms + Timbal memory... 553.1 KB peak + Timbal burst (100 concurrent)... p50=660.7 ms + Timbal throughput conc=1... 60/s + Timbal throughput conc=10... 104/s + Timbal throughput conc=50... 136/s + Timbal throughput conc=200... 135/s + PG bare latency... p50=527.1 µs + PG bare memory... 16.8 KB peak + PG bare burst (100 concurrent)... p50=543.2 µs + PG bare throughput conc=1... 1729/s + PG bare throughput conc=10... 1733/s + PG bare throughput conc=50... 1732/s + PG bare throughput conc=200... 1682/s [Phase 1 · rounds=64] - Timbal latency... p50=51.3 ms - Timbal memory... 1.2 MB peak - Timbal burst (100 concurrent)... p50=4139.5 ms - Timbal throughput conc=1... 19/s - Timbal throughput conc=10... 26/s - Timbal throughput conc=50... 23/s - Timbal throughput conc=200... 21/s - PG bare latency... p50=1.11 ms - PG bare memory... 18.3 KB peak - PG bare burst (100 concurrent)... p50=1.11 ms - PG bare throughput conc=1... 893/s - PG bare throughput conc=10... 892/s - PG bare throughput conc=50... 890/s - PG bare throughput conc=200... 883/s + Timbal latency... p50=26.7 ms + Timbal memory... 1.0 MB peak + Timbal burst (100 concurrent)... p50=1694.7 ms + Timbal throughput conc=1... 30/s + Timbal throughput conc=10... 55/s + Timbal throughput conc=50... 55/s + Timbal throughput conc=200... 55/s + PG bare latency... p50=1.05 ms + PG bare memory... 17.4 KB peak + PG bare burst (100 concurrent)... p50=1.07 ms + PG bare throughput conc=1... 907/s + PG bare throughput conc=10... 895/s + PG bare throughput conc=50... 897/s + PG bare throughput conc=200... 898/s Activating Logfire (send_to_logfire=False)... [Phase 2 · rounds=8] - PG+Logfire latency... p50=2.19 ms - PG+Logfire memory... 313.3 KB peak - PG+Logfire burst (100 concurrent)... p50=2.16 ms - PG+Logfire throughput conc=1... 456/s - PG+Logfire throughput conc=10... 455/s - PG+Logfire throughput conc=50... 455/s - PG+Logfire throughput conc=200... 451/s + PG+Logfire latency... p50=2.20 ms + PG+Logfire memory... 50.2 KB peak + PG+Logfire burst (100 concurrent)... p50=2.39 ms + PG+Logfire throughput conc=1... 335/s + PG+Logfire throughput conc=10... 436/s + PG+Logfire throughput conc=50... 437/s + PG+Logfire throughput conc=200... 432/s [Phase 2 · rounds=16] - PG+Logfire latency... p50=4.24 ms - PG+Logfire memory... 312.4 KB peak - PG+Logfire burst (100 concurrent)... p50=4.04 ms - PG+Logfire throughput conc=1... 244/s - PG+Logfire throughput conc=10... 243/s - PG+Logfire throughput conc=50... 243/s - PG+Logfire throughput conc=200... 224/s + PG+Logfire latency... p50=4.27 ms + PG+Logfire memory... 51.0 KB peak + PG+Logfire burst (100 concurrent)... p50=3.95 ms + PG+Logfire throughput conc=1... 236/s + PG+Logfire throughput conc=10... 235/s + PG+Logfire throughput conc=50... 235/s + PG+Logfire throughput conc=200... 230/s [Phase 2 · rounds=32] - PG+Logfire latency... p50=8.03 ms - PG+Logfire memory... 312.1 KB peak - PG+Logfire burst (100 concurrent)... p50=7.91 ms - PG+Logfire throughput conc=1... 125/s - PG+Logfire throughput conc=10... 123/s - PG+Logfire throughput conc=50... 126/s - PG+Logfire throughput conc=200... 122/s + PG+Logfire latency... p50=8.16 ms + PG+Logfire memory... 49.4 KB peak + PG+Logfire burst (100 concurrent)... p50=7.92 ms + PG+Logfire throughput conc=1... 123/s + PG+Logfire throughput conc=10... 113/s + PG+Logfire throughput conc=50... 122/s + PG+Logfire throughput conc=200... 119/s [Phase 2 · rounds=64] - PG+Logfire latency... p50=15.6 ms - PG+Logfire memory... 311.0 KB peak - PG+Logfire burst (100 concurrent)... p50=15.4 ms - PG+Logfire throughput conc=1... 64/s - PG+Logfire throughput conc=10... 59/s - PG+Logfire throughput conc=50... 59/s - PG+Logfire throughput conc=200... 59/s + PG+Logfire latency... p50=15.5 ms + PG+Logfire memory... 49.0 KB peak + PG+Logfire burst (100 concurrent)... p50=15.2 ms + PG+Logfire throughput conc=1... 63/s + PG+Logfire throughput conc=10... 65/s + PG+Logfire throughput conc=50... 62/s + PG+Logfire throughput conc=200... 64/s ────────────────────────────────────────────────────────────────────────────────────────────  8 branch rounds (~24 hot-path nodes/steps) @@ -118,36 +118,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 5.70 ms 155.6 µs 2.24 ms - p50 5.22 ms 146.9 µs 2.19 ms - p95 6.46 ms 205.7 µs 2.50 ms - p99 32.0 ms 299.3 µs 2.65 ms + mean 3.58 ms 142.0 µs 2.30 ms + p50 3.41 ms 137.9 µs 2.20 ms + p95 4.73 ms 161.7 µs 2.73 ms + p99 5.87 ms 221.2 µs 3.15 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 182.3 KB 933 B - PG bare 16.4 KB 84 B - PG+Logfire 313.3 KB 1604 B + Timbal 162.5 KB 832 B + PG bare 17.8 KB 91 B + PG+Logfire 50.2 KB 257 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 337.3 ms 148.8 µs 2.16 ms - p75 338.5 ms 154.8 µs 2.22 ms - p95 339.7 ms 192.0 µs 2.43 ms - p99 340.1 ms 315.3 µs 2.53 ms - max 340.1 ms 315.3 µs 2.53 ms + p50 137.0 ms 141.0 µs 2.39 ms + p75 137.7 ms 144.6 µs 2.82 ms + p95 149.0 ms 170.7 µs 3.85 ms + p99 149.3 ms 290.9 µs 5.34 ms + max 149.3 ms 290.9 µs 5.34 ms - wall: Timbal: 340.1 ms | PG bare: 0.3 ms | PG+Logfire: 2.5 ms + wall: Timbal: 149.3 ms | PG bare: 0.3 ms | PG+Logfire: 5.3 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 175/s 6414/s 456/s - 10 284/s 6526/s 455/s - 50 273/s 6596/s 455/s - 200 245/s 6473/s 451/s + 1 257/s 6025/s 335/s + 10 656/s 5962/s 436/s + 50 768/s 6016/s 437/s + 200 717/s 6051/s 432/s ────────────────────────────────────────────────────────────────────────────────────────────  16 branch rounds (~48 hot-path nodes/steps) @@ -156,36 +156,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 12.1 ms 294.7 µs 4.24 ms - p50 11.3 ms 285.7 µs 4.24 ms - p95 13.3 ms 370.8 µs 4.54 ms - p99 55.1 ms 413.6 µs 4.70 ms + mean 6.28 ms 279.0 µs 4.32 ms + p50 5.75 ms 268.4 µs 4.27 ms + p95 8.11 ms 349.0 µs 5.00 ms + p99 12.7 ms 415.3 µs 5.63 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 336.8 KB 1724 B - PG bare 15.8 KB 81 B - PG+Logfire 312.4 KB 1600 B + Timbal 293.6 KB 1503 B + PG bare 16.6 KB 85 B + PG+Logfire 51.0 KB 261 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 836.4 ms 283.2 µs 4.04 ms - p75 838.5 ms 293.4 µs 4.15 ms - p95 840.3 ms 373.4 µs 4.31 ms - p99 840.7 ms 457.9 µs 4.67 ms - max 840.7 ms 457.9 µs 4.67 ms + p50 307.0 ms 276.0 µs 3.95 ms + p75 308.0 ms 284.5 µs 4.31 ms + p95 308.9 ms 403.1 µs 4.54 ms + p99 320.0 ms 495.5 µs 4.77 ms + max 320.0 ms 495.5 µs 4.77 ms - wall: Timbal: 840.7 ms | PG bare: 0.5 ms | PG+Logfire: 4.7 ms + wall: Timbal: 320.0 ms | PG bare: 0.5 ms | PG+Logfire: 4.8 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 81/s 3389/s 244/s - 10 127/s 2279/s 243/s - 50 125/s 3411/s 243/s - 200 105/s 3417/s 224/s + 1 153/s 3201/s 236/s + 10 291/s 3200/s 235/s + 50 324/s 3172/s 235/s + 200 302/s 3241/s 230/s ────────────────────────────────────────────────────────────────────────────────────────────  32 branch rounds (~96 hot-path nodes/steps) @@ -194,36 +194,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 25.0 ms 574.0 µs 8.07 ms - p50 23.8 ms 559.2 µs 8.03 ms - p95 27.0 ms 670.7 µs 8.59 ms - p99 62.4 ms 808.6 µs 9.23 ms + mean 16.2 ms 548.6 µs 8.24 ms + p50 15.4 ms 527.1 µs 8.16 ms + p95 20.4 ms 686.4 µs 8.90 ms + p99 34.7 ms 752.3 µs 11.5 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 635.5 KB 3254 B - PG bare 15.9 KB 81 B - PG+Logfire 312.1 KB 1598 B + Timbal 553.1 KB 2832 B + PG bare 16.8 KB 86 B + PG+Logfire 49.4 KB 253 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 1719.9 ms 565.4 µs 7.91 ms - p75 1723.5 ms 583.8 µs 8.06 ms - p95 1726.0 ms 682.6 µs 8.51 ms - p99 1726.7 ms 752.7 µs 8.72 ms - max 1726.7 ms 752.7 µs 8.72 ms + p50 660.7 ms 543.2 µs 7.92 ms + p75 662.8 ms 565.4 µs 8.15 ms + p95 664.1 ms 663.3 µs 8.96 ms + p99 674.6 ms 794.5 µs 9.64 ms + max 674.6 ms 794.5 µs 9.64 ms - wall: Timbal: 1726.7 ms | PG bare: 0.8 ms | PG+Logfire: 8.7 ms + wall: Timbal: 674.6 ms | PG bare: 0.8 ms | PG+Logfire: 9.6 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 38/s 1750/s 125/s - 10 57/s 1765/s 123/s - 50 56/s 1738/s 126/s - 200 48/s 1750/s 122/s + 1 60/s 1729/s 123/s + 10 104/s 1733/s 113/s + 50 136/s 1732/s 122/s + 200 135/s 1682/s 119/s ────────────────────────────────────────────────────────────────────────────────────────────  64 branch rounds (~192 hot-path nodes/steps) @@ -232,36 +232,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 54.9 ms 1.13 ms 15.7 ms - p50 51.3 ms 1.11 ms 15.6 ms - p95 92.0 ms 1.23 ms 16.4 ms - p99 117.5 ms 1.29 ms 17.5 ms + mean 27.9 ms 1.09 ms 15.5 ms + p50 26.7 ms 1.05 ms 15.5 ms + p95 36.2 ms 1.26 ms 16.4 ms + p99 42.7 ms 1.32 ms 17.7 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 1.2 MB 6083 B - PG bare 18.3 KB 94 B - PG+Logfire 311.0 KB 1592 B + Timbal 1.0 MB 5475 B + PG bare 17.4 KB 89 B + PG+Logfire 49.0 KB 251 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 4139.5 ms 1.11 ms 15.4 ms - p75 4181.8 ms 1.13 ms 15.5 ms - p95 4187.8 ms 1.20 ms 15.8 ms - p99 4189.1 ms 1.28 ms 16.4 ms - max 4189.1 ms 1.28 ms 16.4 ms + p50 1694.7 ms 1.07 ms 15.2 ms + p75 1700.5 ms 1.13 ms 15.5 ms + p95 1706.6 ms 1.27 ms 16.2 ms + p99 1719.7 ms 1.43 ms 16.4 ms + max 1719.7 ms 1.43 ms 16.4 ms - wall: Timbal: 4189.1 ms | PG bare: 1.3 ms | PG+Logfire: 16.4 ms + wall: Timbal: 1719.7 ms | PG bare: 1.4 ms | PG+Logfire: 16.4 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 19/s 893/s 64/s - 10 26/s 892/s 59/s - 50 23/s 890/s 59/s - 200 21/s 883/s 59/s + 1 30/s 907/s 63/s + 10 55/s 895/s 65/s + 50 55/s 897/s 62/s + 200 55/s 898/s 64/s ──────────────────────────────────────────────────────────────────────────────────────────── Timbal: bounded branch loop unrolled into explicit workflow steps. diff --git a/benchmarks/pydantic/results/bench_double_fanout.txt b/benchmarks/pydantic/results/bench_double_fanout.txt index 4eb35bff..92cf24a8 100644 --- a/benchmarks/pydantic/results/bench_double_fanout.txt +++ b/benchmarks/pydantic/results/bench_double_fanout.txt @@ -7,70 +7,70 @@ ════════════════════════════════════════════════════════════════════════════════════════ [Phase 1 · N=16] - Timbal trivial... p50=4.62 ms - Timbal async work... p50=6.02 ms - Timbal memory... 277.6 KB peak - Timbal burst (100 concurrent, async work)... p50=516.5 ms - PG bare trivial... p50=203.4 µs - PG bare async work... p50=2.71 ms - PG bare memory... 113.0 KB peak - PG bare burst (100 concurrent, async work)... p50=24.3 ms + Timbal trivial... p50=2.05 ms + Timbal async work... p50=4.02 ms + Timbal memory... 207.2 KB peak + Timbal burst (100 concurrent, async work)... p50=166.6 ms + PG bare trivial... p50=183.8 µs + PG bare async work... p50=2.64 ms + PG bare memory... 44.6 KB peak + PG bare burst (100 concurrent, async work)... p50=19.8 ms [Phase 1 · N=32] - Timbal trivial... p50=8.57 ms - Timbal async work... p50=9.50 ms - Timbal memory... 507.4 KB peak - Timbal burst (100 concurrent, async work)... p50=1091.9 ms - PG bare trivial... p50=275.8 µs - PG bare async work... p50=2.87 ms - PG bare memory... 68.9 KB peak - PG bare burst (100 concurrent, async work)... p50=44.8 ms + Timbal trivial... p50=3.34 ms + Timbal async work... p50=5.15 ms + Timbal memory... 370.4 KB peak + Timbal burst (100 concurrent, async work)... p50=370.5 ms + PG bare trivial... p50=245.9 µs + PG bare async work... p50=2.74 ms + PG bare memory... 66.3 KB peak + PG bare burst (100 concurrent, async work)... p50=34.2 ms [Phase 1 · N=64] - Timbal trivial... p50=16.7 ms - Timbal async work... p50=18.0 ms - Timbal memory... 962.7 KB peak - Timbal burst (100 concurrent, async work)... p50=2532.1 ms - PG bare trivial... p50=406.3 µs - PG bare async work... p50=3.05 ms - PG bare memory... 112.3 KB peak - PG bare burst (100 concurrent, async work)... p50=112.8 ms + Timbal trivial... p50=6.41 ms + Timbal async work... p50=7.53 ms + Timbal memory... 702.0 KB peak + Timbal burst (100 concurrent, async work)... p50=811.0 ms + PG bare trivial... p50=372.4 µs + PG bare async work... p50=2.90 ms + PG bare memory... 110.4 KB peak + PG bare burst (100 concurrent, async work)... p50=184.0 ms [Phase 1 · N=128] - Timbal trivial... p50=36.8 ms - Timbal async work... p50=39.6 ms - Timbal memory... 1.8 MB peak - Timbal burst (100 concurrent, async work)... p50=5827.5 ms - PG bare trivial... p50=798.4 µs - PG bare async work... p50=4.05 ms - PG bare memory... 204.4 KB peak - PG bare burst (100 concurrent, async work)... p50=291.9 ms + Timbal trivial... p50=15.6 ms + Timbal async work... p50=16.5 ms + Timbal memory... 1.3 MB peak + Timbal burst (100 concurrent, async work)... p50=1838.9 ms + PG bare trivial... p50=577.7 µs + PG bare async work... p50=3.36 ms + PG bare memory... 196.2 KB peak + PG bare burst (100 concurrent, async work)... p50=153.7 ms Activating Logfire (send_to_logfire=False)... [Phase 2 · N=16] - PG+Logfire trivial... p50=1.18 ms - PG+Logfire async work... p50=3.72 ms - PG+Logfire memory... 451.3 KB peak - PG+Logfire burst (100 concurrent, async work)... p50=70.9 ms + PG+Logfire trivial... p50=876.6 µs + PG+Logfire async work... p50=3.69 ms + PG+Logfire memory... 72.9 KB peak + PG+Logfire burst (100 concurrent, async work)... p50=60.9 ms [Phase 2 · N=32] - PG+Logfire trivial... p50=1.06 ms - PG+Logfire async work... p50=3.72 ms - PG+Logfire memory... 405.5 KB peak - PG+Logfire burst (100 concurrent, async work)... p50=90.3 ms + PG+Logfire trivial... p50=919.7 µs + PG+Logfire async work... p50=3.77 ms + PG+Logfire memory... 92.8 KB peak + PG+Logfire burst (100 concurrent, async work)... p50=78.2 ms [Phase 2 · N=64] - PG+Logfire trivial... p50=1.09 ms - PG+Logfire async work... p50=3.82 ms - PG+Logfire memory... 449.1 KB peak - PG+Logfire burst (100 concurrent, async work)... p50=168.9 ms + PG+Logfire trivial... p50=1.07 ms + PG+Logfire async work... p50=3.72 ms + PG+Logfire memory... 139.6 KB peak + PG+Logfire burst (100 concurrent, async work)... p50=105.2 ms [Phase 2 · N=128] - PG+Logfire trivial... p50=1.48 ms - PG+Logfire async work... p50=4.62 ms - PG+Logfire memory... 537.8 KB peak - PG+Logfire burst (100 concurrent, async work)... p50=336.7 ms + PG+Logfire trivial... p50=1.28 ms + PG+Logfire async work... p50=4.32 ms + PG+Logfire memory... 224.1 KB peak + PG+Logfire burst (100 concurrent, async work)... p50=229.4 ms ────────────────────────────────────────────────────────────────────────────────────────  Width N=16 (2N+3 target steps; PG manual uses 5 graph nodes) @@ -79,36 +79,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 4.78 ms 224.8 µs 1.20 ms - p50 4.62 ms 203.4 µs 1.18 ms - p95 5.14 ms 353.0 µs 1.46 ms - p99 6.72 ms 439.0 µs 1.54 ms + mean 2.16 ms 202.5 µs 903.4 µs + p50 2.05 ms 183.8 µs 876.6 µs + p95 2.62 ms 330.7 µs 1.08 ms + p99 3.01 ms 365.5 µs 1.31 ms Async work branches (2× 1 ms fan-outs) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 6.23 ms 2.84 ms 3.74 ms - p50 6.02 ms 2.71 ms 3.72 ms - p95 6.73 ms 3.73 ms 4.15 ms - p99 7.24 ms 5.42 ms 4.37 ms + mean 4.22 ms 2.67 ms 3.68 ms + p50 4.02 ms 2.64 ms 3.69 ms + p95 4.47 ms 2.88 ms 3.99 ms + p99 13.6 ms 3.19 ms 4.16 ms Burst (100 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 516.5 ms 24.3 ms 70.9 ms - p75 517.5 ms 24.8 ms 73.0 ms - p95 518.5 ms 25.3 ms 73.8 ms - p99 518.6 ms 25.4 ms 74.8 ms - max 518.6 ms 25.4 ms 74.8 ms + p50 166.6 ms 19.8 ms 60.9 ms + p75 167.6 ms 19.9 ms 62.8 ms + p95 178.9 ms 30.0 ms 76.1 ms + p99 179.3 ms 30.1 ms 76.7 ms + max 179.3 ms 30.1 ms 76.7 ms - wall: Timbal: 518.6 ms | PG bare: 25.4 ms | PG+Logfire: 74.8 ms + wall: Timbal: 179.3 ms | PG bare: 30.1 ms | PG+Logfire: 76.7 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 277.6 KB 1421 B - PG bare 113.0 KB 579 B - PG+Logfire 451.3 KB 2311 B + Timbal 207.2 KB 1061 B + PG bare 44.6 KB 228 B + PG+Logfire 72.9 KB 373 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=32 (2N+3 target steps; PG manual uses 5 graph nodes) @@ -117,36 +117,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 9.13 ms 298.2 µs 1.09 ms - p50 8.57 ms 275.8 µs 1.06 ms - p95 9.45 ms 413.7 µs 1.40 ms - p99 43.4 ms 487.0 µs 1.57 ms + mean 3.48 ms 260.2 µs 939.7 µs + p50 3.34 ms 245.9 µs 919.7 µs + p95 3.92 ms 393.5 µs 1.06 ms + p99 9.00 ms 499.8 µs 1.30 ms Async work branches (2× 1 ms fan-outs) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 10.4 ms 2.91 ms 3.73 ms - p50 9.50 ms 2.87 ms 3.72 ms - p95 10.8 ms 3.21 ms 3.99 ms - p99 46.9 ms 3.41 ms 4.09 ms + mean 5.32 ms 2.76 ms 3.77 ms + p50 5.15 ms 2.74 ms 3.77 ms + p95 5.85 ms 2.98 ms 4.11 ms + p99 16.4 ms 3.11 ms 4.38 ms Burst (100 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 1091.9 ms 44.8 ms 90.3 ms - p75 1094.0 ms 46.2 ms 93.9 ms - p95 1095.8 ms 47.0 ms 95.4 ms - p99 1096.3 ms 47.2 ms 96.3 ms - max 1096.3 ms 47.2 ms 96.3 ms + p50 370.5 ms 34.2 ms 78.2 ms + p75 371.4 ms 34.7 ms 79.9 ms + p95 372.9 ms 45.3 ms 92.7 ms + p99 383.3 ms 45.6 ms 93.1 ms + max 383.3 ms 45.6 ms 93.1 ms - wall: Timbal: 1096.3 ms | PG bare: 47.2 ms | PG+Logfire: 96.3 ms + wall: Timbal: 383.3 ms | PG bare: 45.6 ms | PG+Logfire: 93.1 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 507.4 KB 2598 B - PG bare 68.9 KB 353 B - PG+Logfire 405.5 KB 2076 B + Timbal 370.4 KB 1897 B + PG bare 66.3 KB 339 B + PG+Logfire 92.8 KB 475 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=64 (2N+3 target steps; PG manual uses 5 graph nodes) @@ -155,36 +155,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 18.3 ms 429.1 µs 1.12 ms - p50 16.7 ms 406.3 µs 1.09 ms - p95 17.8 ms 550.7 µs 1.26 ms - p99 64.6 ms 656.1 µs 1.47 ms + mean 6.68 ms 405.2 µs 1.38 ms + p50 6.41 ms 372.4 µs 1.07 ms + p95 7.23 ms 576.1 µs 3.65 ms + p99 18.2 ms 787.2 µs 5.12 ms Async work branches (2× 1 ms fan-outs) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 21.1 ms 3.07 ms 4.29 ms - p50 18.0 ms 3.05 ms 3.82 ms - p95 57.2 ms 3.26 ms 8.51 ms - p99 74.3 ms 3.41 ms 13.3 ms + mean 8.09 ms 2.91 ms 3.76 ms + p50 7.53 ms 2.90 ms 3.72 ms + p95 9.43 ms 3.09 ms 4.14 ms + p99 23.9 ms 3.22 ms 4.25 ms Burst (100 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 2532.1 ms 112.8 ms 168.9 ms - p75 2536.4 ms 115.1 ms 171.9 ms - p95 2539.8 ms 117.1 ms 174.4 ms - p99 2540.6 ms 117.5 ms 175.1 ms - max 2540.6 ms 117.5 ms 175.1 ms + p50 811.0 ms 184.0 ms 105.2 ms + p75 812.5 ms 185.1 ms 107.6 ms + p95 814.5 ms 186.1 ms 110.4 ms + p99 825.4 ms 197.1 ms 123.8 ms + max 825.4 ms 197.1 ms 123.8 ms - wall: Timbal: 2540.6 ms | PG bare: 117.5 ms | PG+Logfire: 175.1 ms + wall: Timbal: 825.4 ms | PG bare: 197.1 ms | PG+Logfire: 123.8 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 962.7 KB 4929 B - PG bare 112.3 KB 575 B - PG+Logfire 449.1 KB 2300 B + Timbal 702.0 KB 3594 B + PG bare 110.4 KB 565 B + PG+Logfire 139.6 KB 715 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=128 (2N+3 target steps; PG manual uses 5 graph nodes) @@ -193,36 +193,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 43.4 ms 818.2 µs 1.51 ms - p50 36.8 ms 798.4 µs 1.48 ms - p95 105.6 ms 1.02 ms 1.76 ms - p99 134.8 ms 1.24 ms 2.06 ms + mean 16.5 ms 598.5 µs 1.31 ms + p50 15.6 ms 577.7 µs 1.28 ms + p95 19.1 ms 760.9 µs 1.55 ms + p99 35.9 ms 895.2 µs 1.75 ms Async work branches (2× 1 ms fan-outs) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 51.3 ms 11.5 ms 4.56 ms - p50 39.6 ms 4.05 ms 4.62 ms - p95 117.8 ms 20.7 ms 4.86 ms - p99 141.4 ms 234.3 ms 4.99 ms + mean 17.6 ms 3.39 ms 5.37 ms + p50 16.5 ms 3.36 ms 4.32 ms + p95 21.6 ms 3.72 ms 10.1 ms + p99 36.1 ms 4.02 ms 35.5 ms Burst (100 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 5827.5 ms 291.9 ms 336.7 ms - p75 5872.4 ms 296.3 ms 342.8 ms - p95 5880.1 ms 300.0 ms 347.6 ms - p99 5883.1 ms 300.6 ms 348.7 ms - max 5883.1 ms 300.6 ms 348.7 ms + p50 1838.9 ms 153.7 ms 229.4 ms + p75 1841.7 ms 156.3 ms 234.9 ms + p95 1844.0 ms 158.3 ms 239.0 ms + p99 1856.2 ms 169.9 ms 253.2 ms + max 1856.2 ms 169.9 ms 253.2 ms - wall: Timbal: 5883.1 ms | PG bare: 300.6 ms | PG+Logfire: 348.7 ms + wall: Timbal: 1856.2 ms | PG bare: 169.9 ms | PG+Logfire: 253.2 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 1.8 MB 9411 B - PG bare 204.4 KB 1047 B - PG+Logfire 537.8 KB 2753 B + Timbal 1.3 MB 6972 B + PG bare 196.2 KB 1005 B + PG+Logfire 224.1 KB 1148 B ──────────────────────────────────────────────────────────────────────────────────────── Timbal: 2N+3 first-class workflow steps, one task/span per step. diff --git a/benchmarks/pydantic/results/bench_linear_loop.txt b/benchmarks/pydantic/results/bench_linear_loop.txt index 58483dfb..8c04aece 100644 --- a/benchmarks/pydantic/results/bench_linear_loop.txt +++ b/benchmarks/pydantic/results/bench_linear_loop.txt @@ -9,106 +9,106 @@ Timbal: ✓ | PG: ✓ [Phase 1 · rounds=8] - Timbal latency... p50=1.44 ms - Timbal memory... 75.1 KB peak - Timbal burst (100 concurrent)... p50=83.2 ms - Timbal throughput conc=1... 591/s - Timbal throughput conc=10... 975/s - Timbal throughput conc=50... 976/s - Timbal throughput conc=200... 947/s - PG bare latency... p50=53.9 µs - PG bare memory... 16.3 KB peak - PG bare burst (100 concurrent)... p50=53.8 µs - PG bare throughput conc=1... 16808/s - PG bare throughput conc=10... 16908/s - PG bare throughput conc=50... 16710/s - PG bare throughput conc=200... 16621/s + Timbal latency... p50=818.2 µs + Timbal memory... 55.5 KB peak + Timbal burst (100 concurrent)... p50=27.6 ms + Timbal throughput conc=1... 1365/s + Timbal throughput conc=10... 2174/s + Timbal throughput conc=50... 2587/s + Timbal throughput conc=200... 2697/s + PG bare latency... p50=52.6 µs + PG bare memory... 16.2 KB peak + PG bare burst (100 concurrent)... p50=54.1 µs + PG bare throughput conc=1... 12780/s + PG bare throughput conc=10... 12797/s + PG bare throughput conc=50... 12932/s + PG bare throughput conc=200... 13011/s [Phase 1 · rounds=16] - Timbal latency... p50=3.26 ms - Timbal memory... 115.3 KB peak - Timbal burst (100 concurrent)... p50=169.6 ms - Timbal throughput conc=1... 294/s - Timbal throughput conc=10... 498/s - Timbal throughput conc=50... 495/s - Timbal throughput conc=200... 459/s - PG bare latency... p50=100.6 µs - PG bare memory... 15.8 KB peak - PG bare burst (100 concurrent)... p50=100.0 µs - PG bare throughput conc=1... 9378/s - PG bare throughput conc=10... 9345/s - PG bare throughput conc=50... 9319/s - PG bare throughput conc=200... 9310/s + Timbal latency... p50=1.34 ms + Timbal memory... 79.3 KB peak + Timbal burst (100 concurrent)... p50=67.1 ms + Timbal throughput conc=1... 725/s + Timbal throughput conc=10... 1282/s + Timbal throughput conc=50... 1514/s + Timbal throughput conc=200... 1532/s + PG bare latency... p50=95.1 µs + PG bare memory... 16.6 KB peak + PG bare burst (100 concurrent)... p50=95.5 µs + PG bare throughput conc=1... 8365/s + PG bare throughput conc=10... 8461/s + PG bare throughput conc=50... 8538/s + PG bare throughput conc=200... 8521/s [Phase 1 · rounds=32] - Timbal latency... p50=6.70 ms - Timbal memory... 199.7 KB peak - Timbal burst (100 concurrent)... p50=379.1 ms - Timbal throughput conc=1... 134/s - Timbal throughput conc=10... 214/s - Timbal throughput conc=50... 235/s - Timbal throughput conc=200... 208/s - PG bare latency... p50=190.9 µs - PG bare memory... 16.5 KB peak - PG bare burst (100 concurrent)... p50=191.7 µs - PG bare throughput conc=1... 5009/s - PG bare throughput conc=10... 5020/s - PG bare throughput conc=50... 5027/s - PG bare throughput conc=200... 5089/s + Timbal latency... p50=2.91 ms + Timbal memory... 145.9 KB peak + Timbal burst (100 concurrent)... p50=130.4 ms + Timbal throughput conc=1... 301/s + Timbal throughput conc=10... 590/s + Timbal throughput conc=50... 692/s + Timbal throughput conc=200... 785/s + PG bare latency... p50=181.6 µs + PG bare memory... 16.7 KB peak + PG bare burst (100 concurrent)... p50=183.0 µs + PG bare throughput conc=1... 4830/s + PG bare throughput conc=10... 4879/s + PG bare throughput conc=50... 4777/s + PG bare throughput conc=200... 4805/s [Phase 1 · rounds=64] - Timbal latency... p50=13.7 ms - Timbal memory... 365.0 KB peak - Timbal burst (100 concurrent)... p50=873.4 ms - Timbal throughput conc=1... 64/s - Timbal throughput conc=10... 91/s - Timbal throughput conc=50... 100/s - Timbal throughput conc=200... 92/s - PG bare latency... p50=376.3 µs - PG bare memory... 15.7 KB peak - PG bare burst (100 concurrent)... p50=379.7 µs - PG bare throughput conc=1... 2499/s - PG bare throughput conc=10... 2547/s - PG bare throughput conc=50... 2540/s - PG bare throughput conc=200... 2494/s + Timbal latency... p50=6.84 ms + Timbal memory... 261.8 KB peak + Timbal burst (100 concurrent)... p50=278.1 ms + Timbal throughput conc=1... 138/s + Timbal throughput conc=10... 295/s + Timbal throughput conc=50... 332/s + Timbal throughput conc=200... 330/s + PG bare latency... p50=351.3 µs + PG bare memory... 17.8 KB peak + PG bare burst (100 concurrent)... p50=365.3 µs + PG bare throughput conc=1... 2525/s + PG bare throughput conc=10... 2509/s + PG bare throughput conc=50... 2546/s + PG bare throughput conc=200... 2504/s Activating Logfire (send_to_logfire=False)... [Phase 2 · rounds=8] - PG+Logfire latency... p50=822.0 µs - PG+Logfire memory... 152.2 KB peak - PG+Logfire burst (100 concurrent)... p50=787.4 µs - PG+Logfire throughput conc=1... 1198/s - PG+Logfire throughput conc=10... 1188/s - PG+Logfire throughput conc=50... 1192/s - PG+Logfire throughput conc=200... 1168/s + PG+Logfire latency... p50=817.2 µs + PG+Logfire memory... 45.2 KB peak + PG+Logfire burst (100 concurrent)... p50=861.5 µs + PG+Logfire throughput conc=1... 1139/s + PG+Logfire throughput conc=10... 1012/s + PG+Logfire throughput conc=50... 1070/s + PG+Logfire throughput conc=200... 1081/s [Phase 2 · rounds=16] - PG+Logfire latency... p50=1.43 ms - PG+Logfire memory... 152.0 KB peak - PG+Logfire burst (100 concurrent)... p50=1.57 ms - PG+Logfire throughput conc=1... 577/s - PG+Logfire throughput conc=10... 654/s - PG+Logfire throughput conc=50... 683/s - PG+Logfire throughput conc=200... 625/s + PG+Logfire latency... p50=1.48 ms + PG+Logfire memory... 44.0 KB peak + PG+Logfire burst (100 concurrent)... p50=1.95 ms + PG+Logfire throughput conc=1... 561/s + PG+Logfire throughput conc=10... 335/s + PG+Logfire throughput conc=50... 656/s + PG+Logfire throughput conc=200... 656/s [Phase 2 · rounds=32] - PG+Logfire latency... p50=2.78 ms - PG+Logfire memory... 149.4 KB peak - PG+Logfire burst (100 concurrent)... p50=2.64 ms - PG+Logfire throughput conc=1... 284/s - PG+Logfire throughput conc=10... 338/s - PG+Logfire throughput conc=50... 264/s - PG+Logfire throughput conc=200... 341/s + PG+Logfire latency... p50=2.63 ms + PG+Logfire memory... 45.4 KB peak + PG+Logfire burst (100 concurrent)... p50=2.82 ms + PG+Logfire throughput conc=1... 338/s + PG+Logfire throughput conc=10... 291/s + PG+Logfire throughput conc=50... 340/s + PG+Logfire throughput conc=200... 348/s [Phase 2 · rounds=64] - PG+Logfire latency... p50=5.60 ms - PG+Logfire memory... 148.6 KB peak - PG+Logfire burst (100 concurrent)... p50=5.47 ms - PG+Logfire throughput conc=1... 184/s + PG+Logfire latency... p50=5.21 ms + PG+Logfire memory... 43.0 KB peak + PG+Logfire burst (100 concurrent)... p50=5.19 ms + PG+Logfire throughput conc=1... 185/s PG+Logfire throughput conc=10... 183/s - PG+Logfire throughput conc=50... 185/s - PG+Logfire throughput conc=200... 176/s + PG+Logfire throughput conc=50... 177/s + PG+Logfire throughput conc=200... 173/s ────────────────────────────────────────────────────────────────────────────────────────────  8 linear steps/nodes @@ -117,36 +117,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 1.48 ms 57.2 µs 909.3 µs - p50 1.44 ms 53.9 µs 822.0 µs - p95 1.75 ms 67.8 µs 1.27 ms - p99 2.07 ms 151.5 µs 1.90 ms + mean 1.05 ms 55.4 µs 890.7 µs + p50 818.2 µs 52.6 µs 817.2 µs + p95 2.15 ms 75.6 µs 1.29 ms + p99 4.23 ms 148.6 µs 1.54 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 75.1 KB 385 B - PG bare 16.3 KB 83 B - PG+Logfire 152.2 KB 779 B + Timbal 55.5 KB 284 B + PG bare 16.2 KB 83 B + PG+Logfire 45.2 KB 232 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 83.2 ms 53.8 µs 787.4 µs - p75 83.5 ms 55.5 µs 841.7 µs - p95 84.1 ms 86.6 µs 1.17 ms - p99 84.3 ms 145.0 µs 1.53 ms - max 84.3 ms 145.0 µs 1.53 ms + p50 27.6 ms 54.1 µs 861.5 µs + p75 28.5 ms 57.8 µs 1.46 ms + p95 39.5 ms 90.3 µs 4.70 ms + p99 39.8 ms 253.4 µs 29.4 ms + max 39.8 ms 253.4 µs 29.4 ms - wall: Timbal: 84.3 ms | PG bare: 0.1 ms | PG+Logfire: 1.5 ms + wall: Timbal: 39.8 ms | PG bare: 0.3 ms | PG+Logfire: 29.4 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 591/s 16808/s 1198/s - 10 975/s 16908/s 1188/s - 50 976/s 16710/s 1192/s - 200 947/s 16621/s 1168/s + 1 1365/s 12780/s 1139/s + 10 2174/s 12797/s 1012/s + 50 2587/s 12932/s 1070/s + 200 2697/s 13011/s 1081/s ────────────────────────────────────────────────────────────────────────────────────────────  16 linear steps/nodes @@ -155,36 +155,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 3.32 ms 106.7 µs 1.51 ms - p50 3.26 ms 100.6 µs 1.43 ms - p95 3.77 ms 140.8 µs 1.89 ms - p99 4.10 ms 211.7 µs 2.60 ms + mean 1.47 ms 99.4 µs 1.60 ms + p50 1.34 ms 95.1 µs 1.48 ms + p95 2.01 ms 127.1 µs 2.18 ms + p99 2.50 ms 193.8 µs 3.07 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 115.3 KB 590 B - PG bare 15.8 KB 81 B - PG+Logfire 152.0 KB 778 B + Timbal 79.3 KB 406 B + PG bare 16.6 KB 85 B + PG+Logfire 44.0 KB 225 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 169.6 ms 100.0 µs 1.57 ms - p75 170.8 ms 105.1 µs 1.74 ms - p95 171.6 ms 146.4 µs 2.67 ms - p99 172.1 ms 198.8 µs 3.96 ms - max 172.1 ms 198.8 µs 3.96 ms + p50 67.1 ms 95.5 µs 1.95 ms + p75 67.9 ms 98.3 µs 2.40 ms + p95 78.0 ms 110.8 µs 3.81 ms + p99 78.3 ms 177.0 µs 10.8 ms + max 78.3 ms 177.0 µs 10.8 ms - wall: Timbal: 172.1 ms | PG bare: 0.2 ms | PG+Logfire: 4.0 ms + wall: Timbal: 78.3 ms | PG bare: 0.2 ms | PG+Logfire: 10.8 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 294/s 9378/s 577/s - 10 498/s 9345/s 654/s - 50 495/s 9319/s 683/s - 200 459/s 9310/s 625/s + 1 725/s 8365/s 561/s + 10 1282/s 8461/s 335/s + 50 1514/s 8538/s 656/s + 200 1532/s 8521/s 656/s ────────────────────────────────────────────────────────────────────────────────────────────  32 linear steps/nodes @@ -193,36 +193,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 6.82 ms 199.1 µs 2.92 ms - p50 6.70 ms 190.9 µs 2.78 ms - p95 7.27 ms 243.0 µs 3.57 ms - p99 7.77 ms 328.5 µs 5.27 ms + mean 3.12 ms 185.8 µs 2.73 ms + p50 2.91 ms 181.6 µs 2.63 ms + p95 4.42 ms 207.6 µs 3.14 ms + p99 5.21 ms 334.7 µs 3.56 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 199.7 KB 1022 B - PG bare 16.5 KB 85 B - PG+Logfire 149.4 KB 765 B + Timbal 145.9 KB 747 B + PG bare 16.7 KB 86 B + PG+Logfire 45.4 KB 233 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 379.1 ms 191.7 µs 2.64 ms - p75 380.3 ms 201.5 µs 2.93 ms - p95 381.5 ms 262.6 µs 3.23 ms - p99 382.0 ms 294.2 µs 3.43 ms - max 382.0 ms 294.2 µs 3.43 ms + p50 130.4 ms 183.0 µs 2.82 ms + p75 131.2 ms 185.8 µs 2.92 ms + p95 142.8 ms 203.5 µs 3.22 ms + p99 143.4 ms 294.1 µs 3.54 ms + max 143.4 ms 294.1 µs 3.54 ms - wall: Timbal: 382.0 ms | PG bare: 0.3 ms | PG+Logfire: 3.4 ms + wall: Timbal: 143.4 ms | PG bare: 0.3 ms | PG+Logfire: 3.5 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 134/s 5009/s 284/s - 10 214/s 5020/s 338/s - 50 235/s 5027/s 264/s - 200 208/s 5089/s 341/s + 1 301/s 4830/s 338/s + 10 590/s 4879/s 291/s + 50 692/s 4777/s 340/s + 200 785/s 4805/s 348/s ────────────────────────────────────────────────────────────────────────────────────────────  64 linear steps/nodes @@ -231,36 +231,36 @@ Latency (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 13.9 ms 380.6 µs 5.69 ms - p50 13.7 ms 376.3 µs 5.60 ms - p95 14.7 ms 391.8 µs 6.50 ms - p99 37.7 ms 488.0 µs 7.88 ms + mean 7.11 ms 361.6 µs 5.26 ms + p50 6.84 ms 351.3 µs 5.21 ms + p95 9.34 ms 423.1 µs 5.81 ms + p99 10.7 ms 561.6 µs 6.33 ms Memory (×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 365.0 KB 1869 B - PG bare 15.7 KB 81 B - PG+Logfire 148.6 KB 761 B + Timbal 261.8 KB 1340 B + PG bare 17.8 KB 91 B + PG+Logfire 43.0 KB 220 B Burst (100 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 873.4 ms 379.7 µs 5.47 ms - p75 875.4 ms 392.7 µs 5.75 ms - p95 877.3 ms 498.9 µs 6.52 ms - p99 877.6 ms 579.7 µs 91.0 ms - max 877.6 ms 579.7 µs 91.0 ms + p50 278.1 ms 365.3 µs 5.19 ms + p75 279.1 ms 375.8 µs 5.42 ms + p95 290.3 ms 523.1 µs 6.02 ms + p99 290.8 ms 594.0 µs 6.77 ms + max 290.8 ms 594.0 µs 6.77 ms - wall: Timbal: 877.6 ms | PG bare: 0.6 ms | PG+Logfire: 91.0 ms + wall: Timbal: 290.8 ms | PG bare: 0.6 ms | PG+Logfire: 6.8 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 64/s 2499/s 184/s - 10 91/s 2547/s 183/s - 50 100/s 2540/s 185/s - 200 92/s 2494/s 176/s + 1 138/s 2525/s 185/s + 10 295/s 2509/s 183/s + 50 332/s 2546/s 177/s + 200 330/s 2504/s 173/s ──────────────────────────────────────────────────────────────────────────────────────────── Timbal: bounded linear loop unrolled into explicit async workflow steps. diff --git a/benchmarks/pydantic/results/bench_parallel.txt b/benchmarks/pydantic/results/bench_parallel.txt index 060dfddd..65d107f3 100644 --- a/benchmarks/pydantic/results/bench_parallel.txt +++ b/benchmarks/pydantic/results/bench_parallel.txt @@ -7,86 +7,86 @@ ════════════════════════════════════════════════════════════════════════════════════════ [Phase 1 · N=4] - Timbal trivial... p50=1.05 ms - Timbal async work... p50=2.19 ms - Timbal memory... 74.8 KB peak - Timbal burst (200 concurrent, async work)... p50=166.4 ms - PG bare trivial... p50=82.2 µs - PG bare async work... p50=1.29 ms - PG bare memory... 44.7 KB peak - PG bare burst (200 concurrent, async work)... p50=8.35 ms + Timbal trivial... p50=442.8 µs + Timbal async work... p50=1.62 ms + Timbal memory... 63.2 KB peak + Timbal burst (200 concurrent, async work)... p50=50.5 ms + PG bare trivial... p50=76.5 µs + PG bare async work... p50=1.26 ms + PG bare memory... 31.5 KB peak + PG bare burst (200 concurrent, async work)... p50=7.71 ms [Phase 1 · N=8] - Timbal trivial... p50=1.46 ms - Timbal async work... p50=2.49 ms - Timbal memory... 113.0 KB peak - Timbal burst (200 concurrent, async work)... p50=245.4 ms - PG bare trivial... p50=88.6 µs + Timbal trivial... p50=588.0 µs + Timbal async work... p50=1.70 ms + Timbal memory... 83.3 KB peak + Timbal burst (200 concurrent, async work)... p50=85.5 ms + PG bare trivial... p50=83.8 µs PG bare async work... p50=1.28 ms - PG bare memory... 60.6 KB peak - PG bare burst (200 concurrent, async work)... p50=12.4 ms + PG bare memory... 38.2 KB peak + PG bare burst (200 concurrent, async work)... p50=11.2 ms [Phase 1 · N=16] - Timbal trivial... p50=2.05 ms - Timbal async work... p50=2.80 ms - Timbal memory... 186.0 KB peak - Timbal burst (200 concurrent, async work)... p50=517.2 ms - PG bare trivial... p50=123.4 µs - PG bare async work... p50=1.32 ms - PG bare memory... 86.6 KB peak - PG bare burst (200 concurrent, async work)... p50=21.7 ms + Timbal trivial... p50=864.7 µs + Timbal async work... p50=1.96 ms + Timbal memory... 132.9 KB peak + Timbal burst (200 concurrent, async work)... p50=163.6 ms + PG bare trivial... p50=98.0 µs + PG bare async work... p50=1.31 ms + PG bare memory... 49.9 KB peak + PG bare burst (200 concurrent, async work)... p50=16.1 ms [Phase 1 · N=32] - Timbal trivial... p50=4.12 ms - Timbal async work... p50=4.43 ms - Timbal memory... 334.4 KB peak - Timbal burst (200 concurrent, async work)... p50=1078.5 ms - PG bare trivial... p50=145.4 µs - PG bare async work... p50=1.43 ms - PG bare memory... 69.5 KB peak - PG bare burst (200 concurrent, async work)... p50=72.6 ms + Timbal trivial... p50=1.44 ms + Timbal async work... p50=2.31 ms + Timbal memory... 228.5 KB peak + Timbal burst (200 concurrent, async work)... p50=323.2 ms + PG bare trivial... p50=127.2 µs + PG bare async work... p50=1.37 ms + PG bare memory... 66.7 KB peak + PG bare burst (200 concurrent, async work)... p50=30.2 ms [Phase 1 · N=64] - Timbal trivial... p50=7.56 ms - Timbal async work... p50=8.09 ms - Timbal memory... 619.6 KB peak - Timbal burst (200 concurrent, async work)... p50=2255.7 ms - PG bare trivial... p50=221.0 µs - PG bare async work... p50=1.54 ms - PG bare memory... 112.7 KB peak - PG bare burst (200 concurrent, async work)... p50=166.7 ms + Timbal trivial... p50=2.95 ms + Timbal async work... p50=3.59 ms + Timbal memory... 418.9 KB peak + Timbal burst (200 concurrent, async work)... p50=668.0 ms + PG bare trivial... p50=193.9 µs + PG bare async work... p50=1.69 ms + PG bare memory... 108.9 KB peak + PG bare burst (200 concurrent, async work)... p50=74.2 ms Activating Logfire (send_to_logfire=False)... [Phase 2 · N=4] - PG+Logfire trivial... p50=766.4 µs - PG+Logfire async work... p50=2.01 ms - PG+Logfire memory... 270.8 KB peak - PG+Logfire burst (200 concurrent, async work)... p50=56.8 ms + PG+Logfire trivial... p50=634.0 µs + PG+Logfire async work... p50=1.86 ms + PG+Logfire memory... 57.3 KB peak + PG+Logfire burst (200 concurrent, async work)... p50=53.4 ms [Phase 2 · N=8] - PG+Logfire trivial... p50=619.1 µs - PG+Logfire async work... p50=2.11 ms - PG+Logfire memory... 279.0 KB peak - PG+Logfire burst (200 concurrent, async work)... p50=62.5 ms + PG+Logfire trivial... p50=548.0 µs + PG+Logfire async work... p50=1.86 ms + PG+Logfire memory... 58.4 KB peak + PG+Logfire burst (200 concurrent, async work)... p50=56.4 ms [Phase 2 · N=16] - PG+Logfire trivial... p50=650.0 µs - PG+Logfire async work... p50=2.13 ms - PG+Logfire memory... 308.7 KB peak - PG+Logfire burst (200 concurrent, async work)... p50=70.3 ms + PG+Logfire trivial... p50=571.9 µs + PG+Logfire async work... p50=1.88 ms + PG+Logfire memory... 70.1 KB peak + PG+Logfire burst (200 concurrent, async work)... p50=63.0 ms [Phase 2 · N=32] - PG+Logfire trivial... p50=669.4 µs - PG+Logfire async work... p50=2.05 ms - PG+Logfire memory... 295.7 KB peak - PG+Logfire burst (200 concurrent, async work)... p50=126.8 ms + PG+Logfire trivial... p50=587.9 µs + PG+Logfire async work... p50=1.88 ms + PG+Logfire memory... 90.3 KB peak + PG+Logfire burst (200 concurrent, async work)... p50=76.7 ms [Phase 2 · N=64] - PG+Logfire trivial... p50=724.3 µs - PG+Logfire async work... p50=2.22 ms - PG+Logfire memory... 336.2 KB peak - PG+Logfire burst (200 concurrent, async work)... p50=328.7 ms + PG+Logfire trivial... p50=667.9 µs + PG+Logfire async work... p50=1.95 ms + PG+Logfire memory... 130.8 KB peak + PG+Logfire burst (200 concurrent, async work)... p50=120.4 ms ────────────────────────────────────────────────────────────────────────────────────────  Width N=4 @@ -95,36 +95,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 1.11 ms 92.7 µs 783.2 µs - p50 1.05 ms 82.2 µs 766.4 µs - p95 1.46 ms 150.4 µs 1.12 ms - p99 1.71 ms 206.9 µs 1.53 ms + mean 507.3 µs 81.2 µs 661.6 µs + p50 442.8 µs 76.5 µs 634.0 µs + p95 540.0 µs 105.1 µs 858.6 µs + p99 757.5 µs 204.6 µs 1.09 ms Async work branches (1 ms sleep) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 2.24 ms 1.30 ms 2.07 ms - p50 2.19 ms 1.29 ms 2.01 ms - p95 2.54 ms 1.40 ms 2.49 ms - p99 3.04 ms 1.48 ms 3.58 ms + mean 1.68 ms 1.27 ms 1.87 ms + p50 1.62 ms 1.26 ms 1.86 ms + p95 1.76 ms 1.32 ms 2.09 ms + p99 2.00 ms 1.50 ms 2.23 ms Burst (200 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 166.4 ms 8.35 ms 56.8 ms - p75 170.6 ms 8.60 ms 60.6 ms - p95 171.8 ms 8.95 ms 63.5 ms - p99 171.9 ms 9.05 ms 63.9 ms - max 171.9 ms 9.11 ms 64.3 ms + p50 50.5 ms 7.71 ms 53.4 ms + p75 51.6 ms 7.86 ms 57.0 ms + p95 62.3 ms 16.9 ms 59.2 ms + p99 62.5 ms 16.9 ms 72.4 ms + max 62.6 ms 17.0 ms 72.8 ms - wall: Timbal: 171.9 ms | PG bare: 9.1 ms | PG+Logfire: 64.3 ms + wall: Timbal: 62.6 ms | PG bare: 17.0 ms | PG+Logfire: 72.8 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 74.8 KB 383 B - PG bare 44.7 KB 229 B - PG+Logfire 270.8 KB 1387 B + Timbal 63.2 KB 323 B + PG bare 31.5 KB 161 B + PG+Logfire 57.3 KB 294 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=8 @@ -133,36 +133,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 1.53 ms 97.1 µs 649.7 µs - p50 1.46 ms 88.6 µs 619.1 µs - p95 1.97 ms 134.8 µs 871.8 µs - p99 2.25 ms 217.2 µs 1.08 ms + mean 659.9 µs 87.8 µs 582.2 µs + p50 588.0 µs 83.8 µs 548.0 µs + p95 817.4 µs 103.8 µs 760.9 µs + p99 964.2 µs 196.1 µs 886.5 µs Async work branches (1 ms sleep) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 2.94 ms 1.29 ms 2.11 ms - p50 2.49 ms 1.28 ms 2.11 ms - p95 5.98 ms 1.38 ms 2.37 ms - p99 13.5 ms 1.41 ms 2.51 ms + mean 1.76 ms 1.29 ms 1.88 ms + p50 1.70 ms 1.28 ms 1.86 ms + p95 1.84 ms 1.37 ms 2.10 ms + p99 1.96 ms 1.46 ms 2.23 ms Burst (200 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 245.4 ms 12.4 ms 62.5 ms - p75 246.9 ms 13.4 ms 65.8 ms - p95 248.0 ms 14.1 ms 68.8 ms - p99 248.2 ms 14.2 ms 69.6 ms - max 248.3 ms 14.2 ms 70.0 ms + p50 85.5 ms 11.2 ms 56.4 ms + p75 86.5 ms 11.7 ms 60.2 ms + p95 87.5 ms 22.1 ms 63.4 ms + p99 97.3 ms 22.2 ms 76.1 ms + max 97.5 ms 22.3 ms 76.5 ms - wall: Timbal: 248.3 ms | PG bare: 14.2 ms | PG+Logfire: 70.0 ms + wall: Timbal: 97.5 ms | PG bare: 22.3 ms | PG+Logfire: 76.5 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 113.0 KB 579 B - PG bare 60.6 KB 310 B - PG+Logfire 279.0 KB 1428 B + Timbal 83.3 KB 426 B + PG bare 38.2 KB 196 B + PG+Logfire 58.4 KB 299 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=16 @@ -171,36 +171,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 2.09 ms 135.7 µs 681.9 µs - p50 2.05 ms 123.4 µs 650.0 µs - p95 2.47 ms 206.0 µs 873.3 µs - p99 2.64 ms 237.0 µs 1.44 ms + mean 922.5 µs 103.2 µs 625.7 µs + p50 864.7 µs 98.0 µs 571.9 µs + p95 981.6 µs 127.0 µs 867.7 µs + p99 1.15 ms 195.0 µs 1.05 ms Async work branches (1 ms sleep) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 2.95 ms 1.33 ms 2.10 ms - p50 2.80 ms 1.32 ms 2.13 ms - p95 3.08 ms 1.44 ms 2.30 ms - p99 3.31 ms 1.53 ms 2.45 ms + mean 2.51 ms 1.31 ms 1.91 ms + p50 1.96 ms 1.31 ms 1.88 ms + p95 5.53 ms 1.37 ms 2.18 ms + p99 10.3 ms 1.40 ms 2.28 ms Burst (200 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 517.2 ms 21.7 ms 70.3 ms - p75 519.2 ms 22.6 ms 75.1 ms - p95 520.6 ms 23.9 ms 77.8 ms - p99 520.8 ms 24.1 ms 78.7 ms - max 520.9 ms 24.2 ms 79.0 ms + p50 163.6 ms 16.1 ms 63.0 ms + p75 164.9 ms 16.8 ms 67.0 ms + p95 165.9 ms 17.4 ms 70.6 ms + p99 177.3 ms 26.5 ms 83.0 ms + max 177.5 ms 26.6 ms 83.4 ms - wall: Timbal: 520.9 ms | PG bare: 24.2 ms | PG+Logfire: 79.0 ms + wall: Timbal: 177.5 ms | PG bare: 26.6 ms | PG+Logfire: 83.4 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 186.0 KB 952 B - PG bare 86.6 KB 444 B - PG+Logfire 308.7 KB 1581 B + Timbal 132.9 KB 680 B + PG bare 49.9 KB 255 B + PG+Logfire 70.1 KB 359 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=32 @@ -209,36 +209,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 4.27 ms 161.1 µs 696.1 µs - p50 4.12 ms 145.4 µs 669.4 µs - p95 4.68 ms 242.0 µs 882.9 µs - p99 5.46 ms 320.1 µs 1.03 ms + mean 1.53 ms 135.3 µs 622.8 µs + p50 1.44 ms 127.2 µs 587.9 µs + p95 1.75 ms 169.1 µs 796.8 µs + p99 2.06 ms 270.5 µs 914.2 µs Async work branches (1 ms sleep) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 4.74 ms 1.45 ms 2.08 ms - p50 4.43 ms 1.43 ms 2.05 ms - p95 4.94 ms 1.57 ms 2.32 ms - p99 33.6 ms 1.70 ms 2.73 ms + mean 2.38 ms 1.37 ms 1.94 ms + p50 2.31 ms 1.37 ms 1.88 ms + p95 2.50 ms 1.47 ms 2.24 ms + p99 2.70 ms 1.56 ms 2.44 ms Burst (200 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 1078.5 ms 72.6 ms 126.8 ms - p75 1080.5 ms 74.8 ms 133.0 ms - p95 1082.6 ms 77.0 ms 138.7 ms - p99 1083.1 ms 77.3 ms 139.7 ms - max 1083.2 ms 77.5 ms 140.0 ms + p50 323.2 ms 30.2 ms 76.7 ms + p75 324.9 ms 31.6 ms 81.1 ms + p95 326.5 ms 32.7 ms 85.2 ms + p99 336.6 ms 42.0 ms 97.9 ms + max 336.7 ms 42.0 ms 98.3 ms - wall: Timbal: 1083.2 ms | PG bare: 77.5 ms | PG+Logfire: 140.0 ms + wall: Timbal: 336.7 ms | PG bare: 42.0 ms | PG+Logfire: 98.3 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 334.4 KB 1712 B - PG bare 69.5 KB 356 B - PG+Logfire 295.7 KB 1514 B + Timbal 228.5 KB 1170 B + PG bare 66.7 KB 342 B + PG+Logfire 90.3 KB 462 B ────────────────────────────────────────────────────────────────────────────────────────  Width N=64 @@ -247,36 +247,36 @@ Trivial branches (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 8.03 ms 237.0 µs 748.1 µs - p50 7.56 ms 221.0 µs 724.3 µs - p95 8.03 ms 337.4 µs 921.9 µs - p99 37.1 ms 411.9 µs 1.07 ms + mean 3.15 ms 206.4 µs 716.8 µs + p50 2.95 ms 193.9 µs 667.9 µs + p95 3.56 ms 273.6 µs 944.5 µs + p99 13.2 ms 339.0 µs 1.18 ms Async work branches (1 ms sleep) (×200) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 8.90 ms 1.56 ms 2.23 ms - p50 8.09 ms 1.54 ms 2.22 ms - p95 8.71 ms 1.72 ms 2.48 ms - p99 45.0 ms 1.81 ms 2.67 ms + mean 3.76 ms 2.11 ms 1.99 ms + p50 3.59 ms 1.69 ms 1.95 ms + p95 4.14 ms 3.50 ms 2.22 ms + p99 13.9 ms 4.56 ms 2.32 ms Burst (200 concurrent, async work) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 2255.7 ms 166.7 ms 328.7 ms - p75 2261.9 ms 170.9 ms 337.9 ms - p95 2265.6 ms 180.4 ms 346.4 ms - p99 2266.0 ms 181.0 ms 347.4 ms - max 2266.4 ms 181.2 ms 347.8 ms + p50 668.0 ms 74.2 ms 120.4 ms + p75 670.6 ms 76.6 ms 126.8 ms + p95 672.2 ms 78.6 ms 132.4 ms + p99 683.9 ms 90.2 ms 144.8 ms + max 684.1 ms 90.3 ms 145.2 ms - wall: Timbal: 2266.4 ms | PG bare: 181.2 ms | PG+Logfire: 347.8 ms + wall: Timbal: 684.1 ms | PG bare: 90.3 ms | PG+Logfire: 145.2 ms Memory (trivial, ×200) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 619.6 KB 3173 B - PG bare 112.7 KB 577 B - PG+Logfire 336.2 KB 1722 B + Timbal 418.9 KB 2145 B + PG bare 108.9 KB 557 B + PG+Logfire 130.8 KB 670 B ──────────────────────────────────────────────────────────────────────────────────────── Timbal: N first-class workflow branches, one task/span per branch. diff --git a/benchmarks/pydantic/results/bench_workflow.txt b/benchmarks/pydantic/results/bench_workflow.txt index 7e60efdf..3c206d41 100644 --- a/benchmarks/pydantic/results/bench_workflow.txt +++ b/benchmarks/pydantic/results/bench_workflow.txt @@ -1,3 +1,9 @@ +Downloading botocore (14.8MiB) +Downloading temporalio (12.0MiB) + Downloaded temporalio + Downloaded botocore +Installed 102 packages in 302ms +warning: The package `huggingface-hub==1.24.0` does not have an extra named `inference` ════════════════════════════════════════════════════════════════════════════════════════  Timbal Workflow vs Pydantic Graph — workflow benchmark @@ -10,81 +16,81 @@ Timbal: ✓ | PG: ✓ [Phase 1 · sequential] - Timbal latency... p50=1.07 ms - Timbal memory... 61.3 KB peak - Timbal burst (200 concurrent)... p50=121.85 ms - Timbal throughput conc=1... 852/s - Timbal throughput conc=10... 1436/s - Timbal throughput conc=50... 1340/s - Timbal throughput conc=200... 1199/s - PG bare latency... p50=30.7 µs - PG bare memory... 15.7 KB peak - PG bare burst (200 concurrent)... p50=31.0 µs - PG bare throughput conc=1... 27604/s - PG bare throughput conc=10... 27539/s - PG bare throughput conc=50... 28391/s - PG bare throughput conc=200... 28150/s + Timbal latency... p50=337.9 µs + Timbal memory... 45.0 KB peak + Timbal burst (200 concurrent)... p50=29.93 ms + Timbal throughput conc=1... 1861/s + Timbal throughput conc=10... 3559/s + Timbal throughput conc=50... 4670/s + Timbal throughput conc=200... 4878/s + PG bare latency... p50=31.0 µs + PG bare memory... 16.7 KB peak + PG bare burst (200 concurrent)... p50=30.7 µs + PG bare throughput conc=1... 18571/s + PG bare throughput conc=10... 18417/s + PG bare throughput conc=50... 18421/s + PG bare throughput conc=200... 17725/s [Phase 1 · fanout] - Timbal latency... p50=1.18 ms - Timbal memory... 83.1 KB peak - Timbal burst (200 concurrent)... p50=193.40 ms - Timbal throughput conc=1... 734/s - Timbal throughput conc=10... 965/s - Timbal throughput conc=50... 960/s - Timbal throughput conc=200... 882/s - PG bare latency... p50=26.0 µs - PG bare memory... 16.4 KB peak - PG bare burst (200 concurrent)... p50=25.4 µs - PG bare throughput conc=1... 32750/s - PG bare throughput conc=10... 33350/s - PG bare throughput conc=50... 32645/s - PG bare throughput conc=200... 33183/s + Timbal latency... p50=413.3 µs + Timbal memory... 57.8 KB peak + Timbal burst (200 concurrent)... p50=39.55 ms + Timbal throughput conc=1... 2201/s + Timbal throughput conc=10... 3477/s + Timbal throughput conc=50... 3986/s + Timbal throughput conc=200... 3973/s + PG bare latency... p50=24.4 µs + PG bare memory... 18.1 KB peak + PG bare burst (200 concurrent)... p50=25.3 µs + PG bare throughput conc=1... 20851/s + PG bare throughput conc=10... 19234/s + PG bare throughput conc=50... 20382/s + PG bare throughput conc=200... 20869/s [Phase 1 · diamond] - Timbal latency... p50=1.02 ms - Timbal memory... 68.5 KB peak - Timbal burst (200 concurrent)... p50=150.50 ms - Timbal throughput conc=1... 951/s - Timbal throughput conc=10... 1364/s - Timbal throughput conc=50... 1261/s - Timbal throughput conc=200... 1149/s - PG bare latency... p50=25.3 µs - PG bare memory... 15.9 KB peak - PG bare burst (200 concurrent)... p50=25.2 µs - PG bare throughput conc=1... 32749/s - PG bare throughput conc=10... 33583/s - PG bare throughput conc=50... 33126/s - PG bare throughput conc=200... 33532/s + Timbal latency... p50=366.0 µs + Timbal memory... 52.8 KB peak + Timbal burst (200 concurrent)... p50=33.14 ms + Timbal throughput conc=1... 2439/s + Timbal throughput conc=10... 4174/s + Timbal throughput conc=50... 4522/s + Timbal throughput conc=200... 4257/s + PG bare latency... p50=24.5 µs + PG bare memory... 16.2 KB peak + PG bare burst (200 concurrent)... p50=25.6 µs + PG bare throughput conc=1... 20421/s + PG bare throughput conc=10... 20772/s + PG bare throughput conc=50... 20237/s + PG bare throughput conc=200... 20163/s Activating Logfire (send_to_logfire=False)... [Phase 2 · sequential] - PG+Logfire latency... p50=592.3 µs - PG+Logfire memory... 295.3 KB peak - PG+Logfire burst (200 concurrent)... p50=587.7 µs - PG+Logfire throughput conc=1... 1652/s - PG+Logfire throughput conc=10... 1639/s - PG+Logfire throughput conc=50... 1626/s - PG+Logfire throughput conc=200... 1605/s + PG+Logfire latency... p50=572.1 µs + PG+Logfire memory... 47.2 KB peak + PG+Logfire burst (200 concurrent)... p50=563.7 µs + PG+Logfire throughput conc=1... 1668/s + PG+Logfire throughput conc=10... 1666/s + PG+Logfire throughput conc=50... 1667/s + PG+Logfire throughput conc=200... 1679/s [Phase 2 · fanout] - PG+Logfire latency... p50=465.2 µs - PG+Logfire memory... 241.0 KB peak - PG+Logfire burst (200 concurrent)... p50=480.3 µs - PG+Logfire throughput conc=1... 1875/s - PG+Logfire throughput conc=10... 2005/s - PG+Logfire throughput conc=50... 1890/s - PG+Logfire throughput conc=200... 1402/s + PG+Logfire latency... p50=455.3 µs + PG+Logfire memory... 45.5 KB peak + PG+Logfire burst (200 concurrent)... p50=454.4 µs + PG+Logfire throughput conc=1... 2029/s + PG+Logfire throughput conc=10... 2033/s + PG+Logfire throughput conc=50... 2009/s + PG+Logfire throughput conc=200... 2029/s [Phase 2 · diamond] - PG+Logfire latency... p50=484.9 µs - PG+Logfire memory... 240.5 KB peak - PG+Logfire burst (200 concurrent)... p50=481.0 µs - PG+Logfire throughput conc=1... 1872/s - PG+Logfire throughput conc=10... 1996/s - PG+Logfire throughput conc=50... 2003/s - PG+Logfire throughput conc=200... 1888/s + PG+Logfire latency... p50=462.5 µs + PG+Logfire memory... 45.1 KB peak + PG+Logfire burst (200 concurrent)... p50=462.5 µs + PG+Logfire throughput conc=1... 1996/s + PG+Logfire throughput conc=10... 2022/s + PG+Logfire throughput conc=50... 1987/s + PG+Logfire throughput conc=200... 1852/s ────────────────────────────────────────────────────────────────────────────────────────  sequential: A → B → C → D @@ -93,36 +99,36 @@ Latency (×200 sequential runs) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 1.11 ms 32.8 µs 611.6 µs - p50 1.07 ms 30.7 µs 592.3 µs - p95 1.31 ms 39.5 µs 746.8 µs - p99 1.69 ms 125.0 µs 872.8 µs + mean 404.9 µs 33.0 µs 585.6 µs + p50 337.9 µs 31.0 µs 572.1 µs + p95 457.0 µs 46.0 µs 674.8 µs + p99 639.9 µs 72.2 µs 936.2 µs Memory (×200 runs) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 61.3 KB 314 B - PG bare 15.7 KB 80 B - PG+Logfire 295.3 KB 1512 B + Timbal 45.0 KB 230 B + PG bare 16.7 KB 86 B + PG+Logfire 47.2 KB 242 B Burst (200 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 121.85 ms 31.0 µs 587.7 µs - p75 122.69 ms 32.3 µs 623.2 µs - p95 123.50 ms 36.6 µs 765.4 µs - p99 123.61 ms 80.5 µs 871.1 µs - max 123.64 ms 108.0 µs 875.0 µs + p50 29.93 ms 30.7 µs 563.7 µs + p75 30.85 ms 32.0 µs 584.7 µs + p95 42.15 ms 34.7 µs 685.7 µs + p99 56.44 ms 92.0 µs 752.5 µs + max 56.72 ms 134.6 µs 904.7 µs - wall: Timbal: 123.6 ms | PG bare: 0.1 ms | PG+Logfire: 0.9 ms + wall: Timbal: 56.7 ms | PG bare: 0.1 ms | PG+Logfire: 0.9 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 852/s 27604/s 1652/s - 10 1436/s 27539/s 1639/s - 50 1340/s 28391/s 1626/s - 200 1199/s 28150/s 1605/s + 1 1861/s 18571/s 1668/s + 10 3559/s 18417/s 1666/s + 50 4670/s 18421/s 1667/s + 200 4878/s 17725/s 1679/s ────────────────────────────────────────────────────────────────────────────────────────  fanout: A → [B, C, D] → E @@ -131,36 +137,36 @@ Latency (×200 sequential runs) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 1.22 ms 27.9 µs 495.1 µs - p50 1.18 ms 26.0 µs 465.2 µs - p95 1.46 ms 30.7 µs 662.5 µs - p99 1.65 ms 92.1 µs 756.1 µs + mean 483.6 µs 25.1 µs 464.5 µs + p50 413.3 µs 24.4 µs 455.3 µs + p95 562.3 µs 25.6 µs 550.0 µs + p99 878.6 µs 56.7 µs 616.9 µs Memory (×200 runs) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 83.1 KB 425 B - PG bare 16.4 KB 84 B - PG+Logfire 241.0 KB 1234 B + Timbal 57.8 KB 296 B + PG bare 18.1 KB 93 B + PG+Logfire 45.5 KB 233 B Burst (200 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 193.40 ms 25.4 µs 480.3 µs - p75 193.91 ms 26.5 µs 607.5 µs - p95 194.79 ms 30.4 µs 2.97 ms - p99 194.96 ms 58.7 µs 5.56 ms - max 195.10 ms 117.3 µs 6.47 ms + p50 39.55 ms 25.3 µs 454.4 µs + p75 40.40 ms 26.1 µs 465.4 µs + p95 50.49 ms 28.0 µs 525.3 µs + p99 50.68 ms 71.4 µs 658.0 µs + max 50.83 ms 99.7 µs 718.2 µs - wall: Timbal: 195.1 ms | PG bare: 0.1 ms | PG+Logfire: 6.5 ms + wall: Timbal: 50.8 ms | PG bare: 0.1 ms | PG+Logfire: 0.7 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 734/s 32750/s 1875/s - 10 965/s 33350/s 2005/s - 50 960/s 32645/s 1890/s - 200 882/s 33183/s 1402/s + 1 2201/s 20851/s 2029/s + 10 3477/s 19234/s 2033/s + 50 3986/s 20382/s 2009/s + 200 3973/s 20869/s 2029/s ────────────────────────────────────────────────────────────────────────────────────────  diamond: A → [B, C] → D @@ -169,36 +175,36 @@ Latency (×200 sequential runs) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - mean 1.05 ms 26.9 µs 524.3 µs - p50 1.02 ms 25.3 µs 484.9 µs - p95 1.28 ms 29.5 µs 748.9 µs - p99 1.48 ms 118.6 µs 899.5 µs + mean 434.4 µs 26.1 µs 471.9 µs + p50 366.0 µs 24.5 µs 462.5 µs + p95 490.5 µs 28.8 µs 542.9 µs + p99 1.08 ms 110.0 µs 617.2 µs Memory (×200 runs) framework peak per run ──────────────────── ──────────── ──────────── - Timbal 68.5 KB 351 B - PG bare 15.9 KB 81 B - PG+Logfire 240.5 KB 1231 B + Timbal 52.8 KB 270 B + PG bare 16.2 KB 83 B + PG+Logfire 45.1 KB 231 B Burst (200 concurrent) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - p50 150.50 ms 25.2 µs 481.0 µs - p75 156.06 ms 26.1 µs 506.7 µs - p95 156.50 ms 32.3 µs 749.6 µs - p99 156.61 ms 69.0 µs 984.6 µs - max 156.64 ms 107.0 µs 989.4 µs + p50 33.14 ms 25.6 µs 462.5 µs + p75 34.31 ms 26.3 µs 484.6 µs + p95 45.56 ms 29.1 µs 582.8 µs + p99 45.75 ms 61.0 µs 761.5 µs + max 45.88 ms 113.7 µs 770.8 µs - wall: Timbal: 156.6 ms | PG bare: 0.1 ms | PG+Logfire: 1.0 ms + wall: Timbal: 45.9 ms | PG bare: 0.1 ms | PG+Logfire: 0.8 ms Throughput (500 loops) Timbal PG bare PG+Logfire ──────────── ────────────── ────────────── ────────────── - 1 951/s 32749/s 1872/s - 10 1364/s 33583/s 1996/s - 50 1261/s 33126/s 2003/s - 200 1149/s 33532/s 1888/s + 1 2439/s 20421/s 1996/s + 10 4174/s 20772/s 2022/s + 50 4522/s 20237/s 1987/s + 200 4257/s 20163/s 1852/s ──────────────────────────────────────────────────────────────────────────────────────── Timbal: Workflow DAG with one span per step; fan-out branches are scheduled by the framework. diff --git a/pyproject.toml b/pyproject.toml index e1444a34..a8ad725b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ requires-python = ">=3.11" dependencies = [ - "anthropic>=0.45.0", + "anthropic>=0.83.0", "httpx>=0.28.0", "mcp>=1.26.0", "openai>=1.60.0", diff --git a/python/tests/codegen/test_cli_lazy.py b/python/tests/codegen/test_cli_lazy.py new file mode 100644 index 00000000..71a0e003 --- /dev/null +++ b/python/tests/codegen/test_cli_lazy.py @@ -0,0 +1,76 @@ +"""CLI startup laziness: libcst must only load for transformer operations. + +The __main__ op table lets --help and lightweight ops (get-flow, get-tools, +get-models, test, evals) dispatch without importing the transformer modules +(each imports libcst at module level, ~160ms of CLI startup). +""" + +import subprocess +import sys + +from timbal.codegen.__main__ import _TRANSFORMER_OPS, _requested_operation + + +class TestOpTableSync: + def test_table_matches_modules_on_disk(self): + """The static table must list exactly the transformer modules.""" + from timbal.codegen.transformers import load_modules + + on_disk = set(load_modules().keys()) + in_table = {module for module, _ in _TRANSFORMER_OPS.values()} + assert in_table == on_disk + + def test_cli_names_map_to_module_names(self): + for cli_name, (module_name, help_line) in _TRANSFORMER_OPS.items(): + assert cli_name.replace("-", "_") == module_name + assert help_line + + +class TestRequestedOperation: + def test_simple(self): + assert _requested_operation(["add-mcp", "--name", "x"]) == "add-mcp" + + def test_global_flag_values_are_not_operations(self): + assert _requested_operation(["--path", "some-dir", "add-mcp"]) == "add-mcp" + assert _requested_operation(["--path=some-dir", "add-tool"]) == "add-tool" + + def test_no_operation(self): + assert _requested_operation(["--help"]) is None + assert _requested_operation([]) is None + + +class TestHelpDoesNotImportLibcst: + def test_top_level_help_lists_ops_without_libcst(self): + code = ( + "import sys\n" + "sys.argv = ['timbal-codegen', '--help']\n" + "from timbal.codegen.__main__ import main\n" + "try:\n" + " main()\n" + "except SystemExit:\n" + " pass\n" + "print('LIBCST_IMPORTED', 'libcst' in sys.modules)\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr + # Help must still list every transformer op (stub parsers). + for op in _TRANSFORMER_OPS: + assert op in result.stdout + assert "LIBCST_IMPORTED False" in result.stdout + + def test_transformer_subcommand_help_registers_full_parser(self): + code = ( + "import sys\n" + "sys.argv = ['timbal-codegen', 'add-mcp', '--help']\n" + "from timbal.codegen.__main__ import main\n" + "try:\n" + " main()\n" + "except SystemExit:\n" + " pass\n" + "print('LIBCST_IMPORTED', 'libcst' in sys.modules)\n" + ) + result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr + # Full parser for the requested op: its specific flags must show. + assert "--transport" in result.stdout or "--name" in result.stdout + assert "LIBCST_IMPORTED True" in result.stdout diff --git a/python/tests/core/test_collector_interrupt_salvage.py b/python/tests/collectors/test_collector_interrupt_salvage.py similarity index 100% rename from python/tests/core/test_collector_interrupt_salvage.py rename to python/tests/collectors/test_collector_interrupt_salvage.py diff --git a/python/tests/conftest.py b/python/tests/conftest.py index c61b00f0..69190455 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -34,16 +34,20 @@ def reset_platform_config_cache(request): if is_integration: _cl._cached_default_config = None _cl._default_config_resolved = False + _cl._file_config_cache.clear() yield _cl._cached_default_config = None _cl._default_config_resolved = False + _cl._file_config_cache.clear() return _cl._cached_default_config = None _cl._default_config_resolved = True + _cl._file_config_cache.clear() yield _cl._cached_default_config = None _cl._default_config_resolved = True + _cl._file_config_cache.clear() @pytest.fixture(autouse=True) diff --git a/python/tests/core/llm/__init__.py b/python/tests/core/llm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/core/llm/test_lazy_provider_sdks.py b/python/tests/core/llm/test_lazy_provider_sdks.py new file mode 100644 index 00000000..b019470a --- /dev/null +++ b/python/tests/core/llm/test_lazy_provider_sdks.py @@ -0,0 +1,100 @@ +"""Provider SDK laziness: openai/anthropic must only load when actually used. + +`from timbal import Agent` imports neither SDK; the first LLM call imports +only the SDK of the provider in use; error classification never imports an +SDK (an exception can only come from an already-imported one). + +Each test runs in a subprocess so sys.modules assertions are not polluted by +the rest of the suite. +""" + +import subprocess +import sys + + +def _run(code: str) -> str: + """Run code in a subprocess; return the LAST stdout line (skips log lines).""" + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + timeout=120, + ) + assert result.returncode == 0, f"stdout={result.stdout}\nstderr={result.stderr}" + lines = [line for line in result.stdout.splitlines() if line.strip()] + return lines[-1] if lines else "" + + +class TestLazyProviderSdks: + def test_agent_import_loads_neither_sdk(self): + out = _run( + "import sys\n" + "from timbal import Agent\n" + "print('openai' in sys.modules, 'anthropic' in sys.modules)\n" + ) + assert out.strip() == "False False" + + def test_testmodel_agent_run_loads_neither_sdk(self): + out = _run( + "import asyncio, sys\n" + "from timbal import Agent\n" + "from timbal.core.test_model import TestModel\n" + "async def main():\n" + " agent = Agent(name='x', model=TestModel(responses=['ok']), tools=[])\n" + " res = await agent(prompt='hi').collect()\n" + " assert res.status.code == 'success', res.error\n" + "asyncio.run(main())\n" + "print('openai' in sys.modules, 'anthropic' in sys.modules)\n" + ) + assert out.strip() == "False False" + + def test_anthropic_client_resolution_does_not_import_openai(self): + out = _run( + "import sys\n" + "from timbal.core.llm import _PROVIDERS, _resolve_client\n" + "from timbal.state.context import RunContext\n" + "ctx = RunContext(tracing_provider=None)\n" + "client, _ = _resolve_client('anthropic', _PROVIDERS['anthropic'], 'sk-fake', None, ctx)\n" + "assert type(client).__name__ == 'AsyncAnthropic'\n" + "print('openai' in sys.modules, 'anthropic' in sys.modules)\n" + ) + assert out.strip() == "False True" + + def test_openai_client_resolution_does_not_import_anthropic(self): + out = _run( + "import sys\n" + "from timbal.core.llm import _PROVIDERS, _resolve_client\n" + "from timbal.state.context import RunContext\n" + "ctx = RunContext(tracing_provider=None)\n" + "client, _ = _resolve_client('openai', _PROVIDERS['openai'], 'sk-fake', None, ctx)\n" + "assert type(client).__name__ == 'AsyncOpenAI'\n" + "print('openai' in sys.modules, 'anthropic' in sys.modules)\n" + ) + assert out.strip() == "True False" + + def test_error_classification_with_single_sdk(self): + """Classifying an anthropic error must work without importing openai.""" + out = _run( + "import sys\n" + "import httpx\n" + "import anthropic\n" + "from timbal.core.fallback_model import is_retryable_provider_error\n" + "resp = httpx.Response(429, request=httpx.Request('POST', 'http://x'))\n" + "exc = anthropic.RateLimitError('rate limited', response=resp, body=None)\n" + "assert is_retryable_provider_error(exc) is True\n" + "assert is_retryable_provider_error(ValueError('nope')) is False\n" + "print('openai' in sys.modules)\n" + ) + assert out.strip() == "False" + + def test_error_classification_with_no_sdk(self): + """TestModel-only processes classify errors with empty class tuples.""" + out = _run( + "import sys\n" + "from timbal.core.provider_errors import provider_error_classes\n" + "classes = provider_error_classes()\n" + "assert all(v == () for v in classes.values()), classes\n" + "assert not isinstance(ValueError('x'), classes['rate_limit'])\n" + "print('openai' in sys.modules, 'anthropic' in sys.modules)\n" + ) + assert out.strip() == "False False" diff --git a/python/tests/core/test_llm_platform_proxy.py b/python/tests/core/llm/test_llm_platform_proxy.py similarity index 98% rename from python/tests/core/test_llm_platform_proxy.py rename to python/tests/core/llm/test_llm_platform_proxy.py index 4c5e6a11..ba304b9f 100644 --- a/python/tests/core/test_llm_platform_proxy.py +++ b/python/tests/core/llm/test_llm_platform_proxy.py @@ -1,6 +1,6 @@ """Tests for LLM platform proxy configuration. -IMPORTANT: These tests require a .env.test_llm_platform_proxy file in the tests/core/ directory. +IMPORTANT: These tests require a .env.test_llm_platform_proxy file in this directory. Copy .env.test_llm_platform_proxy.example to .env.test_llm_platform_proxy and configure the following variables: - TIMBAL_API_HOST (e.g., api.timbal.ai) - TIMBAL_ORG_ID (your organization ID) @@ -38,6 +38,7 @@ def load_test_env(): return True +@pytest.mark.integration @pytest.mark.skipif(not TEST_ENV_FILE.exists(), reason=SKIP_REASON) class TestPlatformProxy: """Test that agents use platform proxy when configured.""" diff --git a/python/tests/core/test_llm_retry.py b/python/tests/core/llm/test_llm_retry.py similarity index 97% rename from python/tests/core/test_llm_retry.py rename to python/tests/core/llm/test_llm_retry.py index 66b99f3f..a98a7ab0 100644 --- a/python/tests/core/test_llm_retry.py +++ b/python/tests/core/llm/test_llm_retry.py @@ -19,7 +19,7 @@ from openai import ( RateLimitError as OpenAIRateLimitError, ) -from timbal.core.llm_router import _retry_on_error +from timbal.core.llm import _retry_on_error class TestRetryOnError: @@ -247,7 +247,7 @@ async def mock_sleep(delay): await original_sleep(0.001) # Actually sleep a tiny bit with patch("asyncio.sleep", side_effect=mock_sleep): - with patch("timbal.core.llm_router.random.uniform", side_effect=lambda _min, cap: cap) as mock_jitter: + with patch("timbal.core.llm.retry.random.uniform", side_effect=lambda _min, cap: cap) as mock_jitter: with pytest.raises(OpenAIRateLimitError): async for _ in _retry_on_error(always_fails_stream, max_retries=3, retry_delay=1.0, context="Test"): pass @@ -272,7 +272,7 @@ async def mock_sleep(delay): delays.append(delay) with patch("asyncio.sleep", side_effect=mock_sleep): - with patch("timbal.core.llm_router.random.uniform", return_value=0.25): + with patch("timbal.core.llm.retry.random.uniform", return_value=0.25): with pytest.raises(OpenAIRateLimitError): async for _ in _retry_on_error(rate_limited_stream, max_retries=1, retry_delay=1.0, context="Test"): pass @@ -294,7 +294,7 @@ async def mock_sleep(delay): delays.append(delay) with patch("asyncio.sleep", side_effect=mock_sleep): - with patch("timbal.core.llm_router.random.uniform", return_value=0.25): + with patch("timbal.core.llm.retry.random.uniform", return_value=0.25): with pytest.raises(OpenAIRateLimitError): async for _ in _retry_on_error(rate_limited_stream, max_retries=1, retry_delay=1.0, context="Test"): pass diff --git a/python/tests/core/test_llm_router_dispatch.py b/python/tests/core/llm/test_llm_router_dispatch.py similarity index 70% rename from python/tests/core/test_llm_router_dispatch.py rename to python/tests/core/llm/test_llm_router_dispatch.py index 0a0b6d92..ce4a5e3c 100644 --- a/python/tests/core/test_llm_router_dispatch.py +++ b/python/tests/core/llm/test_llm_router_dispatch.py @@ -6,7 +6,7 @@ from anthropic import AsyncAnthropic from openai import AsyncOpenAI from pydantic import BaseModel, SecretStr -from timbal.core.llm_router import _PROVIDERS, _get_client, _resolve_client +from timbal.core.llm import _PROVIDERS, _get_client, _resolve_client async def _empty_async_stream(): @@ -37,7 +37,7 @@ def _make_run_context(platform_config=None): class TestGetClients: def test_get_openai_client_cached(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c1 = _get_client(AsyncOpenAI, "key_a", None, "openai") @@ -45,7 +45,7 @@ def test_get_openai_client_cached(self): assert c1 is c2 def test_get_openai_client_different_keys(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c1 = _get_client(AsyncOpenAI, "key_a", None, "openai") @@ -53,7 +53,7 @@ def test_get_openai_client_different_keys(self): assert c1 is not c2 def test_get_openai_client_with_base_url(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c1 = _get_client(AsyncOpenAI, "key", "https://custom.api.com/v1", "groq") @@ -61,7 +61,7 @@ def test_get_openai_client_with_base_url(self): assert c1 is not c2 def test_get_anthropic_client_cached(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c1 = _get_client(AsyncAnthropic, "key_a", None, "anthropic") @@ -69,7 +69,7 @@ def test_get_anthropic_client_cached(self): assert c1 is c2 def test_get_anthropic_client_different_keys(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c1 = _get_client(AsyncAnthropic, "key_a", None, "anthropic") @@ -148,7 +148,7 @@ class TestLlmRouterProviderValidation: """Test model string parsing and validation in _llm_router.""" async def _collect(self, **kwargs): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router chunks = [] async for chunk in _llm_router(**kwargs): chunks.append(chunk) @@ -156,7 +156,7 @@ async def _collect(self, **kwargs): @pytest.mark.asyncio async def test_missing_provider_prefix_raises(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() with pytest.raises(ValueError, match="provider/model_name"): async for _ in _llm_router(model="gpt-4o"): @@ -164,7 +164,7 @@ async def test_missing_provider_prefix_raises(self): @pytest.mark.asyncio async def test_unsupported_provider_raises(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() with pytest.raises(ValueError, match="Unsupported provider"): async for _ in _llm_router(model="fakeprovider/some-model"): @@ -172,7 +172,7 @@ async def test_unsupported_provider_raises(self): @pytest.mark.asyncio async def test_anthropic_missing_max_tokens_raises(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): with pytest.raises(ValueError, match="max_tokens"): @@ -182,7 +182,7 @@ async def test_anthropic_missing_max_tokens_raises(self): @pytest.mark.asyncio async def test_secretstr_converted_before_use(self): """SecretStr values for api_key and base_url are unwrapped.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client = MagicMock() @@ -193,7 +193,7 @@ async def _empty_stream(): mock_client.messages.create = AsyncMock(return_value=_empty_stream()) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): try: async for _ in _llm_router( model="anthropic/claude-sonnet-4-6", @@ -211,7 +211,7 @@ class TestLlmRouterAnthropicKwargs: @pytest.mark.asyncio async def test_system_prompt_included(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() captured_kwargs = {} @@ -223,7 +223,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.messages.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -239,7 +239,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_temperature_included_when_set(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() captured_kwargs = {} @@ -251,7 +251,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.messages.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -267,7 +267,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_provider_params_forwarded(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() captured_kwargs = {} @@ -279,7 +279,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.messages.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -296,7 +296,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_tools_included_in_kwargs(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() captured_kwargs = {} @@ -311,7 +311,7 @@ async def fake_create(**kwargs): mock_tool = MagicMock() mock_tool.anthropic_schema = {"name": "my_tool", "description": "does stuff", "input_schema": {}} - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -326,13 +326,167 @@ async def fake_create(**kwargs): assert "tools" in captured_kwargs assert captured_kwargs["tools"] == [mock_tool.anthropic_schema] + @pytest.mark.asyncio + async def test_automatic_prompt_caching_default(self): + """Top-level cache_control (automatic prompt caching) is sent by default.""" + from timbal.core.llm import _llm_router + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.messages.create = fake_create + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="anthropic/claude-sonnet-4-6", + max_tokens=100, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert captured_kwargs.get("cache_control") == {"type": "ephemeral"} + + @pytest.mark.asyncio + async def test_automatic_prompt_caching_override_ttl(self): + """provider_params can replace the default cache_control (e.g. 1h TTL).""" + from timbal.core.llm import _llm_router + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.messages.create = fake_create + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="anthropic/claude-sonnet-4-6", + max_tokens=100, + provider_params={"cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert captured_kwargs.get("cache_control") == {"type": "ephemeral", "ttl": "1h"} + + @pytest.mark.asyncio + async def test_automatic_prompt_caching_opt_out(self): + """cache_control=None in provider_params removes the kwarg entirely.""" + from timbal.core.llm import _llm_router + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.messages.create = fake_create + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="anthropic/claude-sonnet-4-6", + max_tokens=100, + provider_params={"cache_control": None}, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert "cache_control" not in captured_kwargs + + @pytest.mark.asyncio + async def test_provider_params_tools_merge_with_client_tools(self): + """Server-side tool defs in provider_params merge with (not clobber) client tools.""" + from timbal.core.llm import _llm_router + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.messages.create = fake_create + + mock_tool = MagicMock() + mock_tool.anthropic_schema = {"name": "my_tool", "description": "does stuff", "input_schema": {}} + server_tool = {"type": "web_search_20250305", "name": "web_search", "max_uses": 3} + + provider_params = {"tools": [server_tool]} + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="anthropic/claude-sonnet-4-6", + max_tokens=100, + tools=[mock_tool], + provider_params=provider_params, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert captured_kwargs["tools"] == [mock_tool.anthropic_schema, server_tool] + # Caller's dict must not be mutated by the pop + assert provider_params == {"tools": [server_tool]} + + @pytest.mark.asyncio + async def test_provider_params_tools_alone(self): + """Server-side tool defs work without any client tools.""" + from timbal.core.llm import _llm_router + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.messages.create = fake_create + + server_tool = {"type": "web_search_20250305", "name": "web_search"} + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="anthropic/claude-sonnet-4-6", + max_tokens=100, + provider_params={"tools": [server_tool]}, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert captured_kwargs["tools"] == [server_tool] + class TestLlmRouterChatCompletionsKwargs: """Test Chat Completions path kwargs (groq, cerebras, etc.).""" @pytest.mark.asyncio async def test_system_prompt_as_system_message(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() captured_kwargs = {} @@ -344,7 +498,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.chat.completions.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): try: async for _ in _llm_router( @@ -359,10 +513,88 @@ async def fake_create(**kwargs): assert messages[0]["role"] == "system" assert messages[0]["content"] == "Be concise." + @pytest.mark.asyncio + async def test_provider_params_tools_merge_with_client_tools(self): + """Extra tool defs in provider_params merge with (not clobber) client tools.""" + from timbal.core.llm import _llm_router + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.chat.completions.create = fake_create + + mock_tool = MagicMock() + mock_tool.openai_chat_completions_schema = {"type": "function", "function": {"name": "my_tool"}} + extra_tool = {"type": "function", "function": {"name": "extra_tool"}} + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="groq/llama-3.3-70b-versatile", + tools=[mock_tool], + provider_params={"tools": [extra_tool]}, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert captured_kwargs["tools"] == [mock_tool.openai_chat_completions_schema, extra_tool] + + @pytest.mark.asyncio + async def test_server_tool_only_turn_dropped_from_messages(self): + """Anthropic memory whose assistant turn is ONLY server-tool blocks must not + produce a bare {"role": "assistant"} message on the chat completions path.""" + from timbal.core.llm import _llm_router + from timbal.types.content import CustomContent, TextContent, ToolUseContent + from timbal.types.message import Message + _make_run_context() + + captured_kwargs = {} + + async def fake_create(**kwargs): + captured_kwargs.update(kwargs) + return _empty_async_stream() + + mock_client = MagicMock() + mock_client.chat.completions.create = fake_create + + memory = [ + Message(role="user", content=[TextContent(text="weather?")]), + Message( + role="assistant", + content=[ + ToolUseContent(id="s1", name="web_search", input={"query": "w"}, is_server_tool_use=True), + CustomContent(value={"type": "web_search_tool_result", "tool_use_id": "s1", "content": []}), + ], + ), + Message(role="user", content=[TextContent(text="and tomorrow?")]), + ] + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): + try: + async for _ in _llm_router( + model="groq/llama-3.3-70b-versatile", + messages=memory, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + sent = captured_kwargs["messages"] + assert all(m.keys() != {"role"} for m in sent), f"bare role dict in {sent}" + assert [m["role"] for m in sent] == ["user", "user"] + @pytest.mark.asyncio async def test_flatten_text_content_for_xiaomi(self): """xiaomi provider has flatten_text_content=True.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.types.content.text import TextContent from timbal.types.message import Message _make_run_context() @@ -378,7 +610,7 @@ async def fake_create(**kwargs): user_msg = Message(role="user", content=[TextContent(text="hello")]) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"XIAOMI_API_KEY": "key"}): try: async for _ in _llm_router( @@ -399,7 +631,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_no_stream_options_when_not_supported(self): """xiaomi has supports_stream_options=False.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() captured_kwargs = {} @@ -411,7 +643,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.chat.completions.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"XIAOMI_API_KEY": "key"}): try: async for _ in _llm_router(model="xiaomi/some-model"): @@ -424,7 +656,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_thinking_omitted_for_providers_without_reasoning_content(self): """Groq path: omit CoT from outbound messages (do not dump into content).""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.types.content.text import TextContent from timbal.types.content.thinking import ThinkingContent from timbal.types.message import Message @@ -443,7 +675,7 @@ async def fake_create(**kwargs): content=[ThinkingContent(thinking="plan"), TextContent(text="ok")], ) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): try: async for _ in _llm_router( @@ -460,7 +692,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_thinking_serialized_as_reasoning_content_for_moonshot(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.types.content.text import TextContent from timbal.types.content.thinking import ThinkingContent from timbal.types.message import Message @@ -479,7 +711,7 @@ async def fake_create(**kwargs): content=[ThinkingContent(thinking="plan"), TextContent(text="ok")], ) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"MOONSHOT_API_KEY": "key"}): try: async for _ in _llm_router( @@ -496,7 +728,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_output_model_adds_response_format(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() class MyOutput(BaseModel): @@ -511,7 +743,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.chat.completions.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): try: async for _ in _llm_router( @@ -572,7 +804,7 @@ class TestLlmRouterTestModelPath: @pytest.mark.asyncio async def test_testmodel_yields_chunks(self): """TestModel.stream() is called directly; no network call occurs.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.core.test_model import TestModel _make_run_context() @@ -588,7 +820,7 @@ async def test_testmodel_yields_chunks(self): @pytest.mark.asyncio async def test_testmodel_increments_call_count(self): """call_count reflects actual calls via the router.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.core.test_model import TestModel _make_run_context() @@ -602,7 +834,7 @@ async def test_testmodel_increments_call_count(self): @pytest.mark.asyncio async def test_testmodel_skips_provider_validation(self): """TestModel short-circuit means no ValueError for missing provider prefix.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.core.test_model import TestModel _make_run_context() @@ -633,7 +865,7 @@ def _make_platform_context(self, app_id=None): @pytest.mark.asyncio async def test_app_id_added_to_headers(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router self._make_platform_context(app_id="app_123") @@ -646,7 +878,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.messages.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): try: async for _ in _llm_router( model="anthropic/claude-sonnet-4-6", @@ -662,7 +894,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_app_id_absent_omits_header(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router self._make_platform_context(app_id=None) @@ -675,7 +907,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.messages.create = fake_create - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): try: async for _ in _llm_router( model="anthropic/claude-sonnet-4-6", @@ -695,7 +927,7 @@ class TestLlmRouterAnthropicStructuredOutput: @pytest.mark.asyncio async def test_output_model(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() @@ -712,7 +944,7 @@ async def fake_stable_create(**kwargs): mock_client.messages.create = fake_stable_create mock_client.beta.messages.create = AsyncMock(side_effect=AssertionError("should not call beta")) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -728,7 +960,7 @@ async def fake_stable_create(**kwargs): @pytest.mark.asyncio async def test_output_model_sets_output_config(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() @@ -745,7 +977,7 @@ async def fake_stable_create(**kwargs): mock_client.messages.create = fake_stable_create mock_client.beta.messages.create = AsyncMock(side_effect=AssertionError("should not call beta")) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -762,7 +994,7 @@ async def fake_stable_create(**kwargs): @pytest.mark.asyncio async def test_no_output_model_uses_stable_endpoint(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() @@ -776,7 +1008,7 @@ async def fake_stable_create(**kwargs): mock_client.messages.create = fake_stable_create mock_client.beta.messages.create = AsyncMock(side_effect=AssertionError("should not call beta")) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -806,14 +1038,14 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_responses_path_system_prompt_as_instructions(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"OPENAI_API_KEY": "key", "TIMBAL_OPENAI_API": "responses"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="openai/gpt-4o", @@ -825,16 +1057,43 @@ async def test_responses_path_system_prompt_as_instructions(self): assert captured_kwargs.get("instructions") == "system prompt" + @pytest.mark.asyncio + async def test_responses_provider_params_tools_merge_with_client_tools(self): + """Server-side tool defs in provider_params merge with (not clobber) client tools.""" + from timbal.core.llm import _llm_router + + _make_run_context() + mock_client, captured_kwargs = self._make_mock_client_and_capturer() + + mock_tool = MagicMock() + mock_tool.openai_responses_schema = {"type": "function", "name": "my_tool", "parameters": {}} + server_tool = {"type": "web_search"} + + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): + with patch.dict(os.environ, {"OPENAI_API_KEY": "key"}): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): + try: + async for _ in _llm_router( + model="openai/gpt-4o", + tools=[mock_tool], + provider_params={"tools": [server_tool]}, + ): + pass + except (RuntimeError, StopAsyncIteration): + pass + + assert captured_kwargs["tools"] == [mock_tool.openai_responses_schema, server_tool] + @pytest.mark.asyncio async def test_responses_path_max_tokens(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"OPENAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="openai/gpt-4o", @@ -849,7 +1108,7 @@ async def test_responses_path_max_tokens(self): @pytest.mark.asyncio async def test_responses_path_output_model(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() @@ -858,9 +1117,9 @@ class MySchema(BaseModel): mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"OPENAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="openai/gpt-4o", @@ -877,7 +1136,7 @@ class MySchema(BaseModel): @pytest.mark.asyncio async def test_responses_path_tools(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() @@ -885,9 +1144,9 @@ async def test_responses_path_tools(self): mock_tool = MagicMock() mock_tool.openai_responses_schema = {"type": "function", "name": "my_tool"} - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"OPENAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="openai/gpt-4o", @@ -902,14 +1161,14 @@ async def test_responses_path_tools(self): @pytest.mark.asyncio async def test_responses_path_temperature(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"OPENAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="openai/gpt-4o", @@ -924,14 +1183,14 @@ async def test_responses_path_temperature(self): @pytest.mark.asyncio async def test_xai_uses_responses_path(self): """xai provider also routes through the Responses API path.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"XAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="xai/grok-3", @@ -963,14 +1222,14 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_max_tokens_as_max_completion_tokens(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "chat_completions"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "chat_completions"): try: async for _ in _llm_router( model="groq/llama-3.3-70b-versatile", @@ -985,14 +1244,14 @@ async def test_max_tokens_as_max_completion_tokens(self): @pytest.mark.asyncio async def test_temperature_forwarded(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "chat_completions"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "chat_completions"): try: async for _ in _llm_router( model="groq/llama-3.3-70b-versatile", @@ -1006,7 +1265,7 @@ async def test_temperature_forwarded(self): @pytest.mark.asyncio async def test_tools_forwarded_with_correct_schema(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() @@ -1017,9 +1276,9 @@ async def test_tools_forwarded_with_correct_schema(self): "function": {"name": "do_thing", "description": "does a thing", "parameters": {}}, } - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "chat_completions"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "chat_completions"): try: async for _ in _llm_router( model="groq/llama-3.3-70b-versatile", @@ -1035,14 +1294,14 @@ async def test_tools_forwarded_with_correct_schema(self): @pytest.mark.asyncio async def test_no_max_tokens_omits_max_completion_tokens(self): """When max_tokens is not provided, max_completion_tokens should be absent.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "chat_completions"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "chat_completions"): try: async for _ in _llm_router(model="groq/llama-3.3-70b-versatile"): pass @@ -1054,14 +1313,14 @@ async def test_no_max_tokens_omits_max_completion_tokens(self): @pytest.mark.asyncio async def test_no_temperature_omits_temperature(self): """When temperature is not passed, it should not appear in kwargs.""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "chat_completions"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "chat_completions"): try: async for _ in _llm_router(model="groq/llama-3.3-70b-versatile"): pass @@ -1077,7 +1336,7 @@ class TestLlmRouterYieldsChunks: @pytest.mark.asyncio async def test_anthropic_path_yields_chunks_with_messages(self): """Cover lines 372-373 (message building) and 412, 415 (yield chunk).""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.types.content.text import TextContent from timbal.types.message import Message @@ -1096,7 +1355,7 @@ async def fake_create(**kwargs): user_msg = Message(role="user", content=[TextContent(text="hello")]) chunks = [] - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "key"}): async for chunk in _llm_router( model="anthropic/claude-sonnet-4-6", @@ -1110,7 +1369,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_openai_responses_path_yields_chunks(self): """Cover lines 458, 461 (yield chunk in responses path).""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() @@ -1126,9 +1385,9 @@ async def fake_create(**kwargs): mock_client.responses.create = fake_create chunks = [] - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"OPENAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): async for chunk in _llm_router( model="openai/gpt-4o", ): @@ -1139,7 +1398,7 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_chat_completions_path_yields_chunks(self): """Cover lines 520, 525 (yield chunk in chat completions path).""" - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() @@ -1155,9 +1414,9 @@ async def fake_create(**kwargs): mock_client.chat.completions.create = fake_create chunks = [] - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"GROQ_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "chat_completions"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "chat_completions"): async for chunk in _llm_router( model="groq/llama-3.3-70b-versatile", ): @@ -1170,7 +1429,7 @@ class TestGetAnthropicClientWithBaseUrl: """Cover _get_client when base_url is provided for Anthropic.""" def test_base_url_set_on_client(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c_with_url = _get_client(AsyncAnthropic, "key_x", "https://custom.api.com/v1", "anthropic") @@ -1179,7 +1438,7 @@ def test_base_url_set_on_client(self): assert c_with_url is not c_without_url def test_base_url_cached_separately(self): - from timbal.core.llm_router import _CLIENT_CACHE + from timbal.core.llm import _CLIENT_CACHE _CLIENT_CACHE.clear() c1 = _get_client(AsyncAnthropic, "key_y", "https://proxy.example.com", "anthropic") diff --git a/python/tests/core/test_agent.py b/python/tests/core/test_agent.py index cb9626bf..f149705a 100644 --- a/python/tests/core/test_agent.py +++ b/python/tests/core/test_agent.py @@ -429,6 +429,44 @@ def valid_tool(x: int) -> int: output = await result.collect() assert isinstance(output, OutputEvent) + @pytest.mark.asyncio + async def test_top_level_agent_inside_tool_handler_does_not_leak_context(self): + """Regression: a TOP-LEVEL agent awaited inside a plain tool handler swaps + in a fresh RunContext (concurrent-sibling logic). With tools iterated + directly on the caller's task, that swap must be restored on exit — + otherwise the caller's next LLM call runs against the wrong trace and + the whole run errors (caught by the delegation benchmarks, not unit + tests: subagents-as-tools have nested paths and inherit the context). + """ + worker = Agent( + name="ctx_worker", + model=TestModel(responses=["worker-done"]), + tools=[], + ) + + async def worker_tool(input: str) -> str: + """Delegate work to the worker agent.""" + result = await worker(prompt=input).collect() + return result.output.collect_text() + + supervisor = Agent( + name="ctx_supervisor", + model=TestModel(responses=[ + Message( + role="assistant", + content=[ToolUseContent(id="c1", name="worker_tool", input={"input": "go"})], + stop_reason="tool_use", + ), + "supervisor-done", + ]), + tools=[worker_tool], + ) + + result = await supervisor(prompt="go").collect() + assert result.error is None, result.error + assert result.status.code == "success" + assert result.output.collect_text() == "supervisor-done" + @pytest.mark.asyncio async def test_hallucinated_tool_does_not_hang(self): """LLM emits a tool_use for a tool that doesn't exist (e.g. a skill name diff --git a/python/tests/core/test_debug2_empty_assistant_text.py b/python/tests/core/test_debug2_empty_assistant_text.py index fcc6bb7b..8ad1a5a3 100644 --- a/python/tests/core/test_debug2_empty_assistant_text.py +++ b/python/tests/core/test_debug2_empty_assistant_text.py @@ -21,7 +21,7 @@ ) from timbal import Agent from timbal.collectors.impl.anthropic import AnthropicCollector -from timbal.core.llm_router import _llm_router +from timbal.core.llm import _llm_router from timbal.core.test_model import TestModel from timbal.state import set_call_id, set_run_context from timbal.state.context import RunContext @@ -232,7 +232,7 @@ async def fake_create(**kwargs): mock_client = MagicMock() mock_client.messages.create = fake_create set_run_context(RunContext(tracing_provider=None)) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "key"}): try: async for _ in _llm_router( @@ -294,7 +294,7 @@ async def rejecting_create(**kwargs): max_tokens=1024, tools=[], ) - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict("os.environ", {"ANTHROPIC_API_KEY": "key"}): turn2 = await agent2( prompt="Dels municipis del valles oriental diguem quins sin els que tenem temes pendents", diff --git a/python/tests/core/test_default_params.py b/python/tests/core/test_default_params.py index d34988ad..bf278e9d 100644 --- a/python/tests/core/test_default_params.py +++ b/python/tests/core/test_default_params.py @@ -1,5 +1,4 @@ import asyncio -import threading import time from datetime import datetime @@ -353,21 +352,23 @@ def handler(message: str, time_str: str) -> str: @pytest.mark.asyncio async def test_parallel_callable_execution(self): - """Test that multiple callable params are executed in parallel.""" + """Test that multiple async callable params are executed in parallel. + + Sync param callables run inline on the event loop (they are cheap + accessors by contract), so concurrent resolution is only observable + with async callables. + """ state = {"active": 0, "overlap": False} - lock = threading.Lock() def make_probe(name: str): - def fn() -> str: - with lock: - state["active"] += 1 - if state["active"] == 2: - state["overlap"] = True - # Yield the thread so the sibling callable can enter while we're still active. - time.sleep(0.05) - with lock: - state["active"] -= 1 + async def fn() -> str: + state["active"] += 1 + if state["active"] == 2: + state["overlap"] = True + # Yield the loop so the sibling callable can enter while we're still active. + await asyncio.sleep(0.05) + state["active"] -= 1 return name return fn diff --git a/python/tests/core/test_event_stream_shape.py b/python/tests/core/test_event_stream_shape.py new file mode 100644 index 00000000..8d62e820 --- /dev/null +++ b/python/tests/core/test_event_stream_shape.py @@ -0,0 +1,266 @@ +"""Golden tests for the exact shape of event streams. + +These pin the number, types, order, and paths of events surfaced when +iterating tools, agents, and workflows — including nested (grandchild) +runnables and DeltaEvent forwarding. They exist so refactors of the event +plumbing (collector wrapping, internal stream consumption, multiplexing) +cannot silently drop, duplicate, or reorder events. + +Conventions pinned here: +- Every runnable invocation surfaces exactly one StartEvent and one + OutputEvent on the consumer's stream, with the runnable's full path. +- Nested runnables' events (children and grandchildren) surface through the + parent's stream, in between the parent's Start and Output. +- Async-generator handlers surface one DeltaEvent per yielded chunk (wrapped + as Custom items when the chunk isn't a DeltaItem), between their Start and + Output, and they propagate through parent agents/workflows. +""" + +import pytest +from timbal import Agent, Tool, Workflow +from timbal.core.test_model import TestModel +from timbal.state import get_run_context +from timbal.types.content import ToolUseContent +from timbal.types.events import DeltaEvent, OutputEvent, StartEvent +from timbal.types.events.delta import Custom +from timbal.types.message import Message + + +def _shape(events) -> list[tuple[str, str]]: + return [(e.type, e.path) for e in events] + + +def _assert_paired(events, path: str, count: int = 1) -> None: + """Assert `path` has exactly `count` Start and Output events, each Start before its Output.""" + starts = [i for i, e in enumerate(events) if e.type == "START" and e.path == path] + outputs = [i for i, e in enumerate(events) if e.type == "OUTPUT" and e.path == path] + assert len(starts) == count, f"{path}: expected {count} StartEvent(s), got {len(starts)}: {_shape(events)}" + assert len(outputs) == count, f"{path}: expected {count} OutputEvent(s), got {len(outputs)}: {_shape(events)}" + for s, o in zip(starts, outputs, strict=True): + assert s < o, f"{path}: OutputEvent at {o} precedes StartEvent at {s}: {_shape(events)}" + + +async def _drain(stream) -> list: + return [event async for event in stream] + + +def _tool_use_msg(*calls: tuple[str, str, dict]) -> Message: + return Message( + role="assistant", + content=[ToolUseContent(id=c[0], name=c[1], input=c[2]) for c in calls], + stop_reason="tool_use", + ) + + +def add(a: int, b: int) -> int: + return a + b + + +async def stream_chunks(prefix: str = "chunk"): + """Async generator handler: yields three plain chunks.""" + yield f"{prefix}_1" + yield f"{prefix}_2" + yield f"{prefix}_3" + + +class TestToolStreamShape: + async def test_plain_tool_exact_shape(self): + tool = Tool(name="add", handler=add) + events = await _drain(tool(a=1, b=2)) + assert _shape(events) == [("START", "add"), ("OUTPUT", "add")] + assert isinstance(events[0], StartEvent) + assert isinstance(events[1], OutputEvent) + assert events[1].output == 3 + assert events[1].status.code == "success" + + async def test_async_gen_tool_exact_shape_with_deltas(self): + tool = Tool(name="streamer", handler=stream_chunks) + events = await _drain(tool()) + assert _shape(events) == [ + ("START", "streamer"), + ("DELTA", "streamer"), + ("DELTA", "streamer"), + ("DELTA", "streamer"), + ("OUTPUT", "streamer"), + ] + deltas = [e for e in events if isinstance(e, DeltaEvent)] + assert all(isinstance(e.item, Custom) for e in deltas) + assert [e.item.data for e in deltas] == ["chunk_1", "chunk_2", "chunk_3"] + + +class TestAgentStreamShape: + def _agent_single_tool_call(self) -> Agent: + return Agent( + name="shape_agent", + model=TestModel(responses=[ + _tool_use_msg(("c1", "add", {"a": 1, "b": 2})), + "The answer is 3.", + ]), + tools=[add], + ) + + async def test_single_tool_call_exact_sequence(self): + events = await _drain(self._agent_single_tool_call()(prompt="add 1 and 2")) + assert _shape(events) == [ + ("START", "shape_agent"), + ("START", "shape_agent.llm"), + ("OUTPUT", "shape_agent.llm"), + ("START", "shape_agent.add"), + ("OUTPUT", "shape_agent.add"), + ("START", "shape_agent.llm"), + ("OUTPUT", "shape_agent.llm"), + ("OUTPUT", "shape_agent"), + ] + tool_output = events[4] + assert tool_output.output == 3 + final = events[-1] + assert isinstance(final, OutputEvent) + assert final.status.code == "success" + + async def test_parallel_tool_calls_completeness(self): + def mul(a: int, b: int) -> int: + return a * b + + agent = Agent( + name="par_agent", + model=TestModel(responses=[ + _tool_use_msg(("c1", "add", {"a": 1, "b": 2}), ("c2", "mul", {"a": 3, "b": 4})), + "Done.", + ]), + tools=[add, mul], + ) + events = await _drain(agent(prompt="compute")) + + # Exact counts per path; interleaving between the two tools is unordered. + _assert_paired(events, "par_agent", count=1) + _assert_paired(events, "par_agent.llm", count=2) + _assert_paired(events, "par_agent.add", count=1) + _assert_paired(events, "par_agent.mul", count=1) + # 2 llm pairs + agent pair + 2 tool pairs = 10 events total. + assert len(events) == 10, _shape(events) + # Agent's Start is first, its Output is last. + assert _shape(events)[0] == ("START", "par_agent") + assert _shape(events)[-1] == ("OUTPUT", "par_agent") + + async def test_delta_forwarding_from_async_gen_tool(self): + streamer = Tool(name="streamer", handler=stream_chunks) + agent = Agent( + name="delta_agent", + model=TestModel(responses=[ + _tool_use_msg(("c1", "streamer", {})), + "Streamed.", + ]), + tools=[streamer], + ) + events = await _drain(agent(prompt="stream")) + + _assert_paired(events, "delta_agent", count=1) + _assert_paired(events, "delta_agent.llm", count=2) + _assert_paired(events, "delta_agent.streamer", count=1) + + deltas = [e for e in events if isinstance(e, DeltaEvent) and e.path == "delta_agent.streamer"] + assert [e.item.data for e in deltas] == ["chunk_1", "chunk_2", "chunk_3"] + # Deltas sit strictly between the tool's Start and Output. + start_i = next(i for i, e in enumerate(events) if e.type == "START" and e.path == "delta_agent.streamer") + output_i = next(i for i, e in enumerate(events) if e.type == "OUTPUT" and e.path == "delta_agent.streamer") + delta_is = [i for i, e in enumerate(events) if isinstance(e, DeltaEvent) and e.path == "delta_agent.streamer"] + assert all(start_i < i < output_i for i in delta_is), _shape(events) + + +class TestWorkflowStreamShape: + async def test_sequential_exact_sequence(self): + def step_a(x: int) -> int: + return x + 1 + + def step_b(x: int) -> int: + return x * 2 + + wf = ( + Workflow(name="seq_wf") + .step(step_a) + .step(step_b, x=lambda: get_run_context().step_span("step_a").output) + ) + events = await _drain(wf(x=1)) + assert _shape(events) == [ + ("START", "seq_wf"), + ("START", "seq_wf.step_a"), + ("OUTPUT", "seq_wf.step_a"), + ("START", "seq_wf.step_b"), + ("OUTPUT", "seq_wf.step_b"), + ("OUTPUT", "seq_wf"), + ] + assert events[-1].output == 4 + + async def test_parallel_steps_completeness(self): + def left(x: int) -> int: + return x + 1 + + def right(x: int) -> int: + return x + 2 + + wf = Workflow(name="par_wf").step(left).step(right) + events = await _drain(wf(x=1)) + + _assert_paired(events, "par_wf", count=1) + _assert_paired(events, "par_wf.left", count=1) + _assert_paired(events, "par_wf.right", count=1) + assert len(events) == 6, _shape(events) + assert _shape(events)[0] == ("START", "par_wf") + assert _shape(events)[-1] == ("OUTPUT", "par_wf") + + async def test_delta_forwarding_from_async_gen_step(self): + wf = Workflow(name="delta_wf").step(Tool(name="streamer", handler=stream_chunks)) + events = await _drain(wf()) + + _assert_paired(events, "delta_wf", count=1) + _assert_paired(events, "delta_wf.streamer", count=1) + deltas = [e for e in events if isinstance(e, DeltaEvent) and e.path == "delta_wf.streamer"] + assert [e.item.data for e in deltas] == ["chunk_1", "chunk_2", "chunk_3"] + assert len(events) == 7, _shape(events) + + +class TestNestedGrandchildStream: + async def test_agent_as_tool_grandchild_events_surface(self): + specialist = Agent( + name="specialist", + model=TestModel(responses=["Specialist done."]), + tools=[], + description="A specialist subagent.", + ) + main = Agent( + name="main_agent", + model=TestModel(responses=[ + _tool_use_msg(("c1", "specialist", {"prompt": "do the thing"})), + "All done.", + ]), + tools=[specialist], + ) + events = await _drain(main(prompt="delegate")) + + _assert_paired(events, "main_agent", count=1) + _assert_paired(events, "main_agent.llm", count=2) + _assert_paired(events, "main_agent.specialist", count=1) + # Grandchild: the subagent's own LLM events surface through the top stream. + _assert_paired(events, "main_agent.specialist.llm", count=1) + # 2 main llm pairs + main pair + specialist pair + specialist llm pair = 10. + assert len(events) == 10, _shape(events) + # Grandchild events are nested strictly inside the subagent's Start/Output window. + sub_start = next(i for i, e in enumerate(events) if e.type == "START" and e.path == "main_agent.specialist") + sub_output = next(i for i, e in enumerate(events) if e.type == "OUTPUT" and e.path == "main_agent.specialist") + for i, e in enumerate(events): + if e.path == "main_agent.specialist.llm": + assert sub_start < i < sub_output, _shape(events) + + async def test_workflow_in_workflow_grandchild_events_surface(self): + def leaf(x: int = 1) -> int: + return x + 41 + + inner = Workflow(name="inner_wf").step(leaf) + outer = Workflow(name="outer_wf").step(inner) + events = await _drain(outer()) + + _assert_paired(events, "outer_wf", count=1) + _assert_paired(events, "outer_wf.inner_wf", count=1) + _assert_paired(events, "outer_wf.inner_wf.leaf", count=1) + assert len(events) == 6, _shape(events) + assert events[-1].output == 42 diff --git a/python/tests/core/test_fallback_model.py b/python/tests/core/test_fallback_model.py index f5d6e8c8..7511e931 100644 --- a/python/tests/core/test_fallback_model.py +++ b/python/tests/core/test_fallback_model.py @@ -4,7 +4,7 @@ from openai import APIStatusError as OpenAIAPIStatusError from timbal import Agent from timbal.core.fallback_model import FallbackModel, ModelEntry -from timbal.core.llm_router import _llm_router +from timbal.core.llm import _llm_router from timbal.errors import FallbackExhausted diff --git a/python/tests/core/test_frontier_models.py b/python/tests/core/test_frontier_models.py index af5bdbac..14f7a4d8 100644 --- a/python/tests/core/test_frontier_models.py +++ b/python/tests/core/test_frontier_models.py @@ -91,14 +91,14 @@ async def fake_create(**kwargs): @pytest.mark.asyncio async def test_grok_45_uses_responses_path(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"XAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="xai/grok-4.5", @@ -113,14 +113,14 @@ async def test_grok_45_uses_responses_path(self): @pytest.mark.asyncio async def test_grok_43_uses_responses_path(self): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"XAI_API_KEY": "key"}): - with patch("timbal.core.llm_router.TIMBAL_OPENAI_API", "responses"): + with patch("timbal.core.llm.router.TIMBAL_OPENAI_API", "responses"): try: async for _ in _llm_router( model="xai/grok-4.3", @@ -156,12 +156,12 @@ async def fake_create(**kwargs): ) @pytest.mark.asyncio async def test_fireworks_models_use_chat_completions(self, model_id: str, api_name: str): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"FIREWORKS_API_KEY": "key"}): try: async for _ in _llm_router(model=model_id, max_tokens=16): @@ -197,12 +197,12 @@ async def fake_create(**kwargs): ) @pytest.mark.asyncio async def test_moonshot_models_use_chat_completions(self, model_id: str, api_name: str): - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router _make_run_context() mock_client, captured_kwargs = self._make_mock_client_and_capturer() - with patch("timbal.core.llm_router._get_client", return_value=mock_client): + with patch("timbal.core.llm.clients._get_client", return_value=mock_client): with patch.dict(os.environ, {"MOONSHOT_API_KEY": "key"}): try: async for _ in _llm_router( diff --git a/python/tests/core/test_frontier_models_integration.py b/python/tests/core/test_frontier_models_integration.py index 03c5827a..05da7af6 100644 --- a/python/tests/core/test_frontier_models_integration.py +++ b/python/tests/core/test_frontier_models_integration.py @@ -98,7 +98,7 @@ async def test_frontier_model_llm_router_streams(model: str, env_key: str, fallb """Direct router smoke test — ensures streaming path works, not only Agent.collect.""" _skip_if_no_key(env_key, fallback_env) - from timbal.core.llm_router import _llm_router + from timbal.core.llm import _llm_router from timbal.types.message import Message from timbal.types.content import TextContent diff --git a/python/tests/core/test_interruptions.py b/python/tests/core/test_interruptions.py index b371b4c0..290628da 100644 --- a/python/tests/core/test_interruptions.py +++ b/python/tests/core/test_interruptions.py @@ -7,6 +7,7 @@ import pytest from timbal import Agent, Tool, Workflow from timbal.core.test_model import TestModel +from timbal.types.content import ToolUseContent from timbal.types.events import OutputEvent from timbal.types.message import Message @@ -74,8 +75,13 @@ def parallel_task_3(duration: float = 1) -> str: @pytest.fixture def long_running_sync_tool(): - """Create a long-running synchronous tool.""" - return Tool(name="long_sync", handler=long_running_sync_handler) + """Create a long-running synchronous tool. + + offload_blocking=True runs the blocking sync handler in the thread pool so + the event loop stays responsive and cancellation can land mid-execution. + Without it, sync handlers run inline and cannot be interrupted mid-flight. + """ + return Tool(name="long_sync", handler=long_running_sync_handler, offload_blocking=True) @pytest.fixture @@ -380,6 +386,84 @@ async def slow_tool() -> str: assert result.status.code == "cancelled" +class TestAgentToolCancellation: + """Regression: cancelling an agent mid-tool must end the RUN as cancelled. + + The single-tool fast path iterates the tool on the agent's own task; the + tool's _stream swallows the CancelledError (recording its own span as + interrupted), so the agent must detect the still-pending cancel request + and re-raise — otherwise the loop continues and the run ends 'success'. + Offline twin of test_key_agent_interruptions::test_tool_interrupt_and_continue + (integration-marked, deselected from default runs), which caught this. + """ + + @pytest.mark.asyncio + async def test_cancel_during_async_tool_via_agent(self): + started = asyncio.Event() + + async def internal_search(query: str) -> str: + started.set() + await asyncio.sleep(10) + return f"Found results for: {query}" + + agent = Agent( + name="cancel_tool_agent", + model=TestModel(responses=[ + Message( + role="assistant", + content=[ToolUseContent(id="c1", name="internal_search", input={"query": "alpha"})], + stop_reason="tool_use", + ), + "Should never get here.", + ]), + tools=[internal_search], + ) + + task = asyncio.create_task(agent(prompt="search alpha").collect()) + await asyncio.wait_for(started.wait(), timeout=2.0) + await asyncio.sleep(0.05) # ensure we're inside the tool's sleep + task.cancel() + result = await task + + assert isinstance(result, OutputEvent) + assert result.status.code == "cancelled", f"got '{result.status.code}'" + assert result.status.reason == "interrupted" + assert result.error is None + # LLM-output salvage: the interrupted turn keeps the tool_use the model issued. + assert result.output is not None + assert any(isinstance(c, ToolUseContent) for c in result.output.content) + + @pytest.mark.asyncio + async def test_agent_usable_after_tool_cancellation(self): + async def slow_tool() -> str: + await asyncio.sleep(10) + return "done" + + agent = Agent( + name="reuse_after_cancel_agent", + model=TestModel(handler=lambda messages: ( + Message( + role="assistant", + content=[ToolUseContent(id="c1", name="slow_tool", input={})], + stop_reason="tool_use", + ) + if len(messages) == 1 + else "follow-up answer" + )), + tools=[slow_tool], + ) + + task = asyncio.create_task(agent(prompt="go").collect()) + await asyncio.sleep(0.1) + task.cancel() + result1 = await task + assert result1.status.code == "cancelled" + + result2 = await agent(prompt="what is 2+2?").collect() + assert result2.status.code == "success" + assert result2.output is not None + + class TestWorkflowInterruption: """Test workflow interruption during step execution.""" @@ -439,11 +523,13 @@ async def step3() -> str: @pytest.mark.asyncio async def test_workflow_interruption_during_parallel_steps(self): """Test workflow interruption when parallel steps are executing.""" + # offload_blocking=True: blocking sync handlers must run in the thread + # pool for the loop to stay responsive and cancellation to land mid-run. workflow = ( Workflow(name="parallel_workflow") - .step(parallel_task_1, duration=1) - .step(parallel_task_2, duration=1) - .step(parallel_task_3, duration=1) + .step(Tool(handler=parallel_task_1, offload_blocking=True), duration=1) + .step(Tool(handler=parallel_task_2, offload_blocking=True), duration=1) + .step(Tool(handler=parallel_task_3, offload_blocking=True), duration=1) ) # Start workflow (all steps run in parallel) diff --git a/python/tests/core/test_memory_compaction.py b/python/tests/core/test_memory_compaction.py index de72792e..b0118094 100644 --- a/python/tests/core/test_memory_compaction.py +++ b/python/tests/core/test_memory_compaction.py @@ -1583,6 +1583,59 @@ async def test_compaction_triggers_at_high_utilization(self, monkeypatch) -> Non # Clean up InMemoryTracingProvider._storage.clear() + @pytest.mark.asyncio + async def test_cached_input_tokens_count_toward_utilization(self, monkeypatch) -> None: + """Anthropic prompt caching moves most input into cache_read/cache_creation + keys (input_tokens only covers the uncached suffix). Utilization must count + them, or compaction would never trigger on cached conversations.""" + from timbal.core.agent import Agent + from timbal.core.memory_compaction import keep_last_n_turns + from timbal.state import set_run_context + from timbal.state.context import RunContext + from timbal.state.tracing.providers import InMemoryTracingProvider + + monkeypatch.setattr("timbal.core.agent.get_context_window", lambda _model: 100_000) + + compaction_called = False + + def tracking_compactor(n): + inner = keep_last_n_turns(n) + + def wrapper(memory): + nonlocal compaction_called + compaction_called = True + return inner(memory) + + return wrapper + + agent = Agent( + name="test_agent", + model=TestModel(), + memory_compaction=tracking_compactor(1), + memory_compaction_ratio=0.75, + ) + + ctx1 = RunContext(tracing_provider=InMemoryTracingProvider) + set_run_context(ctx1) + await agent(prompt="Turn 1").collect() + + # 90% utilization, but almost all of it sits in cache keys — the + # pre-fix predicate saw only 1k input + 4k output = 5% utilization. + root1 = ctx1.root_span() + root1.usage["anthropic/claude-haiku-4-5:input_tokens"] = 1_000 + root1.usage["anthropic/claude-haiku-4-5:cache_read_input_tokens"] = 80_000 + root1.usage["anthropic/claude-haiku-4-5:cache_creation_input_tokens"] = 5_000 + root1.usage["anthropic/claude-haiku-4-5:output_tokens"] = 4_000 + await ctx1._save_trace() + + ctx2 = RunContext(parent_id=ctx1.id, tracing_provider=InMemoryTracingProvider) + set_run_context(ctx2) + await agent(prompt="Turn 2").collect() + + assert compaction_called, "Cache token usage must count toward context utilization" + + InMemoryTracingProvider._storage.clear() + @pytest.mark.asyncio async def test_compaction_skipped_at_low_utilization(self, monkeypatch) -> None: """When previous run used only 10% of context window, compactors should NOT fire.""" diff --git a/python/tests/core/test_runnable.py b/python/tests/core/test_runnable.py index ccbba3e0..6efff85d 100644 --- a/python/tests/core/test_runnable.py +++ b/python/tests/core/test_runnable.py @@ -121,6 +121,112 @@ async def test_serialization(self, simple_tool): assert serialized['name'] == 'simple' +class TestHookIntrospectionIsolation: + """Regression: hook is_coroutine flags must be per instance, not per class. + + They used to be written to ``cls`` by a field validator, so two instances + of the same class with different sync/async hooks clobbered each other — + the last-constructed instance's flag won for both. + """ + + @pytest.mark.asyncio + async def test_sync_and_async_hooks_do_not_clobber_each_other(self): + calls: list[str] = [] + + def sync_hook(): + calls.append("sync") + + async def async_hook(): + calls.append("async") + + def handler(x: str) -> str: + return x + + sync_tool = Tool(name="sync_hooked", handler=handler, pre_hook=sync_hook) + # Constructing the async-hooked tool AFTER must not flip the sync tool's flag. + async_tool = Tool(name="async_hooked", handler=handler, pre_hook=async_hook) + + assert sync_tool._pre_hook_is_coroutine is False + assert async_tool._pre_hook_is_coroutine is True + + result = await sync_tool(x="a").collect() + assert result.error is None + result = await async_tool(x="b").collect() + assert result.error is None + assert calls == ["sync", "async"] + + +class TestBlockingHandlerWarning: + """Sync handlers run inline on the event loop; ones that block past the + threshold must be flagged once with an actionable warning.""" + + @pytest.mark.asyncio + async def test_slow_sync_handler_sets_warned_flag(self): + import time as _time + + def slow_handler() -> str: + _time.sleep(0.15) # > default 100ms threshold + return "done" + + tool = Tool(name="slow_sync", handler=slow_handler) + result = await tool().collect() + assert result.error is None + assert tool._blocking_warned is True + + # Warned once — the flag stays set on subsequent runs (no re-warning). + await tool().collect() + assert tool._blocking_warned is True + + @pytest.mark.asyncio + async def test_fast_sync_handler_does_not_warn(self): + def fast_handler() -> str: + return "done" + + tool = Tool(name="fast_sync", handler=fast_handler) + result = await tool().collect() + assert result.error is None + assert tool._blocking_warned is False + + @pytest.mark.asyncio + async def test_offloaded_handler_does_not_warn(self): + import time as _time + + def slow_handler() -> str: + _time.sleep(0.15) + return "done" + + tool = Tool(name="slow_offloaded", handler=slow_handler, offload_blocking=True) + result = await tool().collect() + assert result.error is None + assert tool._blocking_warned is False + + +class TestSyncGeneratorNoneYield: + """Regression: sync_to_async_gen used None as its end-of-stream sentinel, + silently truncating sync generators that legitimately yield None.""" + + @pytest.mark.asyncio + async def test_sync_generator_yielding_none_is_not_truncated(self): + def gen(): + yield 1 + yield None + yield 3 + + tool = Tool(name="none_yielder", handler=gen) + events = [e async for e in tool()] + # Chunks after the None must survive (the old sentinel ended the stream + # at the None, silently dropping everything behind it)... + deltas = [e for e in events if e.type == "DELTA"] + assert [d.item.data for d in deltas] == [1, 3] + # ...and the final output must retain the None itself. (The None chunk + # isn't surfaced as a streaming delta: the collector protocol treats + # process()->None as "skip" — a separate, purely cosmetic quirk.) + output_event = events[-1] + assert output_event.type == "OUTPUT" + assert output_event.status.code == "success" + assert output_event.output == [1, None, 3] + + class TestErrorHandling: """Test error handling in Runnable execution.""" diff --git a/python/tests/core/test_suspend_cross_process.py b/python/tests/core/test_suspend_cross_process.py index d58bb780..20943ecd 100644 --- a/python/tests/core/test_suspend_cross_process.py +++ b/python/tests/core/test_suspend_cross_process.py @@ -32,18 +32,15 @@ from pathlib import Path import pytest -from pydantic import TypeAdapter from timbal import Agent, Tool, Workflow from timbal.core.test_model import TestModel from timbal.state import RunContext, get_run_context, suspend from timbal.state.tracing.providers.jsonl import JsonlTracingProvider from timbal.state.tracing.trace import Trace from timbal.types.content import ToolUseContent -from timbal.types.events import Event, InteractionEvent, OutputEvent +from timbal.types.events import InteractionEvent, OutputEvent, validate_event from timbal.types.message import Message -_EVENT_ADAPTER = TypeAdapter(Event) - def _interaction_events(events) -> list[InteractionEvent]: return [e for e in events if isinstance(e, InteractionEvent)] @@ -89,7 +86,7 @@ def test_interaction_event_round_trips_through_event_union(self): # model_dump_json -> json.loads -> discriminated-union validate. raw = json.loads(ev.model_dump_json()) - restored = _EVENT_ADAPTER.validate_python(raw) + restored = validate_event(raw) assert isinstance(restored, InteractionEvent) assert restored.type == "INTERACTION" @@ -120,7 +117,7 @@ def test_paused_output_event_round_trips(self): metadata={"type": "Agent"}, ) - restored = _EVENT_ADAPTER.validate_python(json.loads(out.model_dump_json())) + restored = validate_event(json.loads(out.model_dump_json())) assert isinstance(restored, OutputEvent) assert restored.status.code == "cancelled" assert restored.status.reason == "input_required" diff --git a/python/tests/core/test_tool.py b/python/tests/core/test_tool.py index 05d9b862..7a41f499 100644 --- a/python/tests/core/test_tool.py +++ b/python/tests/core/test_tool.py @@ -6,7 +6,7 @@ import pytest from pydantic import BaseModel from timbal import Tool -from timbal.core.llm_router import _llm_router +from timbal.core.llm import _llm_router from timbal.types.events import OutputEvent from timbal.types.message import Message diff --git a/python/tests/fixtures/test.jsonl b/python/tests/fixtures/test.jsonl new file mode 100644 index 00000000..4df473eb --- /dev/null +++ b/python/tests/fixtures/test.jsonl @@ -0,0 +1,2 @@ +{"name": "Alice Smith", "age": 28, "score": 95.5, "join_date": "2023-01-15", "status": "Active"} +{"name": "Bob Johnson", "age": 34, "score": 87.2, "join_date": "2023-03-22", "status": "Inactive"} diff --git a/python/tests/server/test_voice_config.py b/python/tests/server/test_voice_config.py index 9a8e3c74..5b7f9ba0 100644 --- a/python/tests/server/test_voice_config.py +++ b/python/tests/server/test_voice_config.py @@ -22,6 +22,58 @@ from .voice_env import VOICE_ENV_KEYS +@pytest.mark.usefixtures("clear_voice_env") +class TestVoiceWarmupIntended: + """Non-voice deployments must not pre-load ONNX models at server boot. + + Regression: warmup used to run for every Agent app whenever the + timbal[voice] extra was installed (e.g. platform images from timbal[all]), + downloading and loading Smart Turn + Namo + Silero for nothing. + """ + + class _Runnable: + voice_config = None + + def _clear_warmup_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TIMBAL_VOICE_WARMUP", raising=False) + for k in list(os.environ): + if k.startswith("TIMBAL_VOICE_"): + monkeypatch.delenv(k, raising=False) + + def test_no_voice_signals_means_no_warmup(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._clear_warmup_env(monkeypatch) + assert voice_routes.voice_warmup_intended(self._Runnable()) is False + + def test_runnable_voice_config_enables_warmup(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._clear_warmup_env(monkeypatch) + r = self._Runnable() + r.voice_config = {"stt_provider": "elevenlabs"} + assert voice_routes.voice_warmup_intended(r) is True + + def test_voice_env_enables_warmup(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._clear_warmup_env(monkeypatch) + monkeypatch.setenv("TIMBAL_VOICE_LANGUAGE", "en") + assert voice_routes.voice_warmup_intended(self._Runnable()) is True + + def test_elevenlabs_voice_id_enables_warmup(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._clear_warmup_env(monkeypatch) + monkeypatch.setenv("ELEVENLABS_VOICE_ID", "abc") + assert voice_routes.voice_warmup_intended(self._Runnable()) is True + + def test_env_override_forces_warmup(self, monkeypatch: pytest.MonkeyPatch) -> None: + """The playground launcher sets TIMBAL_VOICE_WARMUP=1 for its children.""" + self._clear_warmup_env(monkeypatch) + monkeypatch.setenv("TIMBAL_VOICE_WARMUP", "1") + assert voice_routes.voice_warmup_intended(self._Runnable()) is True + + def test_env_override_forces_no_warmup(self, monkeypatch: pytest.MonkeyPatch) -> None: + self._clear_warmup_env(monkeypatch) + monkeypatch.setenv("TIMBAL_VOICE_WARMUP", "0") + r = self._Runnable() + r.voice_config = {"stt_provider": "elevenlabs"} # even a voice app + assert voice_routes.voice_warmup_intended(r) is False + + @pytest.mark.usefixtures("clear_voice_env") class TestDefaultVoiceConfigFromEnv: def test_defaults_when_unset(self) -> None: diff --git a/python/tests/core/test_parallelism.py b/python/tests/state/test_parallelism.py similarity index 100% rename from python/tests/core/test_parallelism.py rename to python/tests/state/test_parallelism.py diff --git a/python/tests/state/tracing/__init__.py b/python/tests/state/tracing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/state/tracing/exporters/__init__.py b/python/tests/state/tracing/exporters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/core/test_otel_exporter.py b/python/tests/state/tracing/exporters/test_otel_exporter.py similarity index 100% rename from python/tests/core/test_otel_exporter.py rename to python/tests/state/tracing/exporters/test_otel_exporter.py diff --git a/python/tests/state/tracing/providers/__init__.py b/python/tests/state/tracing/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/tests/core/test_in_memory_tracing_provider.py b/python/tests/state/tracing/providers/test_in_memory_tracing_provider.py similarity index 100% rename from python/tests/core/test_in_memory_tracing_provider.py rename to python/tests/state/tracing/providers/test_in_memory_tracing_provider.py diff --git a/python/tests/core/test_jsonl_tracing_provider.py b/python/tests/state/tracing/providers/test_jsonl_tracing_provider.py similarity index 100% rename from python/tests/core/test_jsonl_tracing_provider.py rename to python/tests/state/tracing/providers/test_jsonl_tracing_provider.py diff --git a/python/tests/core/test_platform_tracing_provider.py b/python/tests/state/tracing/providers/test_platform_tracing_provider.py similarity index 97% rename from python/tests/core/test_platform_tracing_provider.py rename to python/tests/state/tracing/providers/test_platform_tracing_provider.py index ca574149..70d872cc 100644 --- a/python/tests/core/test_platform_tracing_provider.py +++ b/python/tests/state/tracing/providers/test_platform_tracing_provider.py @@ -9,7 +9,7 @@ TIMBAL_APP_ID= All tests are skipped automatically if the file is absent. -Run with: uv run pytest python/tests/core/test_platform_tracing_provider.py -v +Run with: uv run pytest python/tests/state/tracing/providers/test_platform_tracing_provider.py -v """ import os @@ -20,7 +20,7 @@ TEST_ENV_FILE = Path(__file__).parent / ".env.test_platform_tracing" SKIP_REASON = ( "Platform tracing integration tests require " - ".env.test_platform_tracing in tests/core/. " + ".env.test_platform_tracing in this directory. " "See the module docstring for required variables." ) diff --git a/python/tests/core/test_sqlite_tracing_provider.py b/python/tests/state/tracing/providers/test_sqlite_tracing_provider.py similarity index 100% rename from python/tests/core/test_sqlite_tracing_provider.py rename to python/tests/state/tracing/providers/test_sqlite_tracing_provider.py diff --git a/python/tests/state/test_trace.py b/python/tests/state/tracing/test_trace.py similarity index 100% rename from python/tests/state/test_trace.py rename to python/tests/state/tracing/test_trace.py diff --git a/python/tests/core/test_tracing.py b/python/tests/state/tracing/test_tracing.py similarity index 100% rename from python/tests/core/test_tracing.py rename to python/tests/state/tracing/test_tracing.py diff --git a/python/tests/core/test_files_integration.py b/python/tests/types/test_files_integration.py similarity index 99% rename from python/tests/core/test_files_integration.py rename to python/tests/types/test_files_integration.py index d7e552eb..e621154b 100644 --- a/python/tests/core/test_files_integration.py +++ b/python/tests/types/test_files_integration.py @@ -45,7 +45,7 @@ def _assert_agent_answer(res: OutputEvent, expected: str) -> None: "openai/gpt-4o-mini", "openai/gpt-4o-mini-responses", "google/gemini-3.1-flash-lite", - "anthropic/claude-sonnet-4-6", + "anthropic/claude-sonnet-5", # ? Add more tests for other models. ] ) diff --git a/python/tests/types/test_message.py b/python/tests/types/test_message.py index 6a6c1d0c..fb7893ea 100644 --- a/python/tests/types/test_message.py +++ b/python/tests/types/test_message.py @@ -166,3 +166,78 @@ def test_message_with_tool_result_to_openai_chat_completions_input() -> None: def test_message_with_tool_result_to_anthropic_input() -> None: message = Message(role="user", content=[ToolResultContent(id="123", content=[TextContent(text="Hello, World!")])]) assert message.to_anthropic_input() == {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "123", "content": [{"type": "text", "text": "Hello, World!"}]}]} + + +# --- Cross-provider replay of server-side tool blocks ----------------------- +# Anthropic memory keeps server_tool_use (ToolUseContent) and +# web_search_tool_result (CustomContent) blocks. When replayed to another +# API shape (e.g. after a fallback-model switch) they must be skipped, not +# raise or leak Anthropic-only block types. + + +def _anthropic_server_tool_message() -> Message: + from timbal.types.content import CustomContent + + return Message( + role="assistant", + content=[ + ToolUseContent(id="srvtoolu_1", name="web_search", input={"query": "weather"}, is_server_tool_use=True), + CustomContent(value={"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": []}), + TextContent(text="It is sunny. [[weather.com](https://weather.com)]"), + ], + ) + + +def test_server_tool_blocks_skipped_in_openai_responses_input() -> None: + inputs = _anthropic_server_tool_message().to_openai_responses_input() + assert inputs == [ + {"role": "assistant", "content": [{"type": "output_text", "text": "It is sunny. [[weather.com](https://weather.com)]"}]} + ] + + +def test_server_tool_blocks_skipped_in_openai_chat_completions_input() -> None: + result = _anthropic_server_tool_message().to_openai_chat_completions_input() + assert result == { + "role": "assistant", + "content": [{"type": "text", "text": "It is sunny. [[weather.com](https://weather.com)]"}], + } + assert "tool_calls" not in result + + +def test_server_tool_only_message_drops_turn_for_openai() -> None: + """An assistant turn with ONLY server-tool blocks (no text) must not become + a bare {"role": "assistant"} dict — OpenAI rejects assistant messages with + neither content nor tool_calls.""" + from timbal.types.content import CustomContent + + message = Message( + role="assistant", + content=[ + ToolUseContent(id="srvtoolu_1", name="web_search", input={"query": "x"}, is_server_tool_use=True), + CustomContent(value={"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": []}), + ], + ) + assert message.to_openai_chat_completions_input() is None + assert message.to_openai_responses_input() == [] + + +def test_thinking_only_message_drops_turn_for_chat_completions() -> None: + """Thinking-only turns serialized with reasoning_as="omit" have no payload either.""" + message = Message(role="assistant", content=[ThinkingContent(thinking="secret plan")]) + assert message.to_openai_chat_completions_input(reasoning_as="omit") is None + # But with reasoning_content round-tripping the turn survives + assert message.to_openai_chat_completions_input(reasoning_as="reasoning_content") == { + "role": "assistant", + "reasoning_content": "secret plan", + } + + +def test_server_tool_blocks_preserved_in_anthropic_input() -> None: + result = _anthropic_server_tool_message().to_anthropic_input() + assert result["content"][0] == { + "type": "server_tool_use", + "id": "srvtoolu_1", + "name": "web_search", + "input": {"query": "weather"}, + } + assert result["content"][1] == {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": []} diff --git a/python/tests/voice/test_session.py b/python/tests/voice/test_session.py index 2bb0090e..6813f060 100644 --- a/python/tests/voice/test_session.py +++ b/python/tests/voice/test_session.py @@ -438,7 +438,7 @@ async def test_warmup_uses_session_model_override(self, monkeypatch: pytest.Monk async def _fake_warmup(model: str) -> None: warmed.append(model) - monkeypatch.setattr("timbal.core.llm_router.warmup_llm_connection", _fake_warmup) + monkeypatch.setattr("timbal.core.llm.warmup_llm_connection", _fake_warmup) agent = Agent(name="t", model="groq/llama-3.1-8b-instant", tools=[]) session = VoiceSession( agent=agent, @@ -458,7 +458,7 @@ async def test_warmup_falls_back_to_agent_model(self, monkeypatch: pytest.Monkey async def _fake_warmup(model: str) -> None: warmed.append(model) - monkeypatch.setattr("timbal.core.llm_router.warmup_llm_connection", _fake_warmup) + monkeypatch.setattr("timbal.core.llm.warmup_llm_connection", _fake_warmup) agent = Agent(name="t", model="groq/llama-3.1-8b-instant", tools=[]) session = VoiceSession(agent=agent, stt=MockSTT(), tts=MockTTS(), turn_detector="heuristic") session._start_llm_warmup() diff --git a/python/timbal/_slots.py b/python/timbal/_slots.py new file mode 100644 index 00000000..be7e5ebd --- /dev/null +++ b/python/timbal/_slots.py @@ -0,0 +1,68 @@ +"""Minimal pydantic-free model machinery for hot-path framework objects. + +Events, ``Span`` and ``RunStatus`` are constructed on every runnable call. +Pydantic's validation, mutable-default deepcopies and ``__getattr__``-routed +private attributes made them a measurable share of per-run overhead, so they +are plain ``__slots__`` classes built on this base instead. The pydantic API +surface consumers rely on (``model_dump()``, ``model_dump_json()``, field +equality) is preserved. +""" + +import json +from typing import Any + +from pydantic import BaseModel + + +def dump_value(value: Any, mode: str = "python") -> Any: + """Recursively convert a value the way pydantic's model_dump would. + + Nested SlotModel/BaseModel instances become dicts in both modes. In + ``json`` mode, values that json.dumps can't handle are stringified. + """ + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, SlotModel): + return value.model_dump(mode=mode) + if isinstance(value, BaseModel): + return value.model_dump(mode=mode) + if isinstance(value, dict): + return {k: dump_value(v, mode) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [dump_value(v, mode) for v in value] + if mode == "json": + return str(value) + return value + + +class SlotModel: + """Base for plain ``__slots__`` classes with a pydantic-compatible surface. + + Subclasses declare ``__slots__`` and list their public fields (in dump + order) in ``_FIELDS``. ``type``-style class-level constants may appear in + ``_FIELDS`` without a slot. + """ + + __slots__ = () + + # Marker for timbal.utils.serialization.dump() — routes through model_dump(). + __timbal_serializable__ = True + + _FIELDS: tuple[str, ...] = () + + def model_dump(self, mode: str = "python", **_kwargs: Any) -> dict[str, Any]: + return {f: dump_value(getattr(self, f, None), mode) for f in self._FIELDS} + + def model_dump_json(self, **_kwargs: Any) -> str: + return json.dumps(self.model_dump(mode="json")) + + def __eq__(self, other: Any) -> bool: + if type(other) is not type(self): + return NotImplemented + return all(getattr(self, f, None) == getattr(other, f, None) for f in self._FIELDS) + + __hash__ = None # Mutable, like pydantic models — unhashable. + + def __repr__(self) -> str: + fields = ", ".join(f"{f}={getattr(self, f, None)!r}" for f in self._FIELDS) + return f"{type(self).__name__}({fields})" diff --git a/python/timbal/codegen/README.md b/python/timbal/codegen/README.md index 9908869e..a37c84a4 100644 --- a/python/timbal/codegen/README.md +++ b/python/timbal/codegen/README.md @@ -576,7 +576,7 @@ When filtering, response includes pagination metadata and full model objects: **Model registry (`models.yaml`):** -Model metadata lives in `python/timbal/models.yaml`. To add or update a model, edit the YAML — it is the single source of truth. The `model=` type annotation in `llm_router.py` is derived from it via `scripts/generate_models.py`: +Model metadata lives in `python/timbal/models.yaml`. To add or update a model, edit the YAML — it is the single source of truth. The `model=` type annotation in `core/llm/router.py` is derived from it via `scripts/generate_models.py`: ```bash uv run python scripts/generate_models.py diff --git a/python/timbal/codegen/__main__.py b/python/timbal/codegen/__main__.py index 91a9d686..d45f2395 100644 --- a/python/timbal/codegen/__main__.py +++ b/python/timbal/codegen/__main__.py @@ -3,6 +3,40 @@ import sys from pathlib import Path +# Static registry of transformer subcommands: cli-name -> (module, help line). +# Lets `--help` list every operation and lightweight ops dispatch WITHOUT +# importing the transformer modules (each imports libcst at module level, +# ~160ms). Only the operation actually invoked registers its full parser. +# A test asserts this table stays in sync with the modules on disk. +_TRANSFORMER_OPS: dict[str, tuple[str, str]] = { + "add-edge": ("add_edge", "Add an ordering or conditional edge between two workflow steps."), + "add-mcp": ("add_mcp", "Add an MCP server to the agent's tools list."), + "add-step": ("add_step", "Add a step to the workflow."), + "add-tool": ("add_tool", "Add a tool to the agent's tools list."), + "remove-edge": ("remove_edge", "Remove an edge between two workflow steps."), + "remove-step": ("remove_step", "Remove a step from the workflow by name."), + "remove-tool": ("remove_tool", "Remove a tool from the agent's tools list by name."), + "set-config": ("set_config", "Set configuration on the agent/step or on a specific tool."), + "set-param": ("set_param", "Set a parameter on a workflow step (static value or mapped from another step's output)."), + "set-position": ("set_position", "Set the (x, y) canvas position for a node, stored in metadata."), +} + + +def _requested_operation(argv: list[str]) -> str | None: + """Extract the subcommand from argv without building the full parser. + + Uses a pre-parser that knows the global flags, so values like + ``--path some-dir add-mcp`` don't get mistaken for the operation. + """ + pre = argparse.ArgumentParser(add_help=False) + pre.add_argument("--path") + pre.add_argument("--dry-run", action="store_true") + _, rest = pre.parse_known_args(argv) + for token in rest: + if not token.startswith("-"): + return token + return None + def main() -> None: parser = argparse.ArgumentParser( @@ -115,16 +149,22 @@ def main() -> None: "completes; 'pretty' is the rich terminal report.", ) - # Defer transformer module loading (pulls in libcst + timbal.codegen which - # are expensive) — only needed for transformer operations, not for - # list-tools, get-flow, or test. - _lightweight_ops = {"get-models", "get-tools", "get-flow", "test", "evals"} - if not (_lightweight_ops & set(sys.argv[1:])): - from timbal.codegen.transformers import load_modules - - transformer_modules = load_modules() - for mod in transformer_modules.values(): - mod.register(subparsers) + # Transformer subcommands register lazily: only the operation actually + # invoked imports its module (and therefore libcst, ~160ms). Everything + # else — including top-level --help — gets stub parsers carrying just the + # help line, which is all argparse needs to list them. + requested = _requested_operation(sys.argv[1:]) + if requested in _TRANSFORMER_OPS: + import importlib + + module_name, _ = _TRANSFORMER_OPS[requested] + importlib.import_module(f"timbal.codegen.transformers.{module_name}").register(subparsers) + for op, (_, help_line) in _TRANSFORMER_OPS.items(): + if op != requested: + subparsers.add_parser(op, help=help_line) + else: + for op, (_, help_line) in _TRANSFORMER_OPS.items(): + subparsers.add_parser(op, help=help_line) args = parser.parse_args() diff --git a/python/timbal/codegen/cst_utils.py b/python/timbal/codegen/cst_utils.py index eae382f6..1c14ef65 100644 --- a/python/timbal/codegen/cst_utils.py +++ b/python/timbal/codegen/cst_utils.py @@ -274,10 +274,7 @@ def has_step_expr(tree: cst.Module, entry_point: str) -> bool: if ( isinstance(item, cst.Expr) and isinstance(item.value, cst.Call) - and isinstance(item.value.func, cst.Attribute) - and isinstance(item.value.func.value, cst.Name) - and item.value.func.value.value == entry_point - and item.value.func.attr.value == "step" + and is_step_call(item.value, entry_point) ): return True return False @@ -436,43 +433,20 @@ def leave_Module( # Add ``from timbal.core import Tool`` if missing. if not has_import(original_node, "timbal.core", "Tool"): - import_insert_idx = 0 - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item_node in stmt.body: - if isinstance(item_node, (cst.Import, cst.ImportFrom)): - import_insert_idx = i + 1 - body.insert(import_insert_idx, cst.parse_statement("from timbal.core import Tool\n")) + insert_imports(body, [cst.parse_statement("from timbal.core import Tool\n")]) # Build ``step_name = Tool(name="step_name", handler=step_name_fn)`` + # and insert it before the entry-point assignment (plain or annotated). + # Also anchor at the first standalone ``.step()`` statement so aliased + # entry points (not visible as assignments) still get the Tool defined + # before its first reference. assignment_code = ( f'{self.step_name} = Tool(name="{self.step_name}", handler={self.step_name}_fn)\n' ) - - # Insert before the entry-point assignment (plain or annotated). Also - # stop at the first standalone ``.step()`` statement so aliased entry - # points (not visible as assignments) still get the Tool defined - # before its first reference. - insert_idx = len(body) - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item_node in stmt.body: - if isinstance(item_node, cst.Assign): - for t in item_node.targets: - if isinstance(t.target, cst.Name) and t.target.value == self.entry_point: - insert_idx = min(insert_idx, i) - elif isinstance(item_node, cst.AnnAssign): - if isinstance(item_node.target, cst.Name) and item_node.target.value == self.entry_point: - insert_idx = min(insert_idx, i) - elif ( - isinstance(item_node, cst.Expr) - and isinstance(item_node.value, cst.Call) - and isinstance(item_node.value.func, cst.Attribute) - and isinstance(item_node.value.func.value, cst.Name) - and item_node.value.func.value.value == self.entry_point - ): - insert_idx = min(insert_idx, i) - body.insert(insert_idx, cst.parse_statement(assignment_code)) + insert_before_assignments( + body, [cst.parse_statement(assignment_code)], + target_names={self.entry_point}, step_calls_of=self.entry_point, + ) return updated_node.with_changes(body=body) @@ -482,6 +456,231 @@ def wrap_bare_function_step(tree: cst.Module, entry_point: str, step_name: str) return tree.visit(_BareFunctionWrapper(entry_point, step_name)) +def is_step_call(call: cst.Call, entry_point: str) -> bool: + """Check if a Call node is ``.step(...)``.""" + return ( + isinstance(call.func, cst.Attribute) + and isinstance(call.func.value, cst.Name) + and call.func.value.value == entry_point + and call.func.attr.value == "step" + ) + + +def step_matches_target( + call: cst.Call, + target: str, + step_names: dict[str, str], + assignments: dict[str, cst.Call] | None = None, +) -> bool: + """Check if a ``.step()`` call's first argument refers to *target*. + + Matches when *target* equals either the step variable name or its runtime + name (from ``step_names``, or resolved from *assignments* when provided). + """ + if not call.args: + return False + first_arg = call.args[0].value + if isinstance(first_arg, cst.Name): + var_name = first_arg.value + if var_name == target: + return True + runtime_name = step_names.get(var_name) + if runtime_name is None and assignments and var_name in assignments: + runtime_name = resolve_runnable_name(assignments[var_name]) + if runtime_name is not None and runtime_name == target: + return True + return False + + +class StepCallRewriter(cst.CSTTransformer): + """Base for transformers that rewrite the target step's ``.step()`` call. + + Subclasses set ``entry_point``, ``target``, ``step_names`` and + ``assignments`` attributes and implement ``_build_step_call_code(call)``. + Sets ``matched = True`` when the target ``.step()`` call was found, so an + unchanged file counts as an idempotent save (see ``apply_operation``). + """ + + def leave_Expr(self, original_node: cst.Expr, updated_node: cst.Expr) -> cst.Expr: # noqa: ARG002 + call = updated_node.value + if not isinstance(call, cst.Call): + return updated_node + if not is_step_call(call, self.entry_point) or not step_matches_target( + call, self.target, self.step_names, self.assignments, + ): + return updated_node + + self.matched = True + new_call = parse_call_statement(self._build_step_call_code(call)) + if new_call is not None: + return updated_node.with_changes(value=new_call) + return updated_node + + +def assignment_resolves_to(assign: cst.Assign | cst.AnnAssign, entry_point: str, name: str) -> bool: + """True when a (non-entry-point) assignment's Call value resolves to *name*. + + Covers plain assignments and annotated assignments (``x: T = Call(...)``). + """ + if not isinstance(assign.value, cst.Call): + return False + if isinstance(assign, cst.AnnAssign): + targets = [assign.target] + else: + targets = [t.target for t in assign.targets] + for target in targets: + if not isinstance(target, cst.Name) or target.value == entry_point: + continue + if resolve_runnable_name(assign.value) == name: + return True + return False + + +def parse_call_statement(code: str) -> cst.Call | None: + """Parse *code* (a single expression statement) and return its Call node.""" + parsed = cst.parse_module(code + "\n") + for stmt in parsed.body: + if isinstance(stmt, cst.SimpleStatementLine): + for item in stmt.body: + if isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): + return item.value + return None + + +def parse_function_def(definition: str) -> cst.FunctionDef: + """Parse a ``--definition`` source string and return its FunctionDef. + + Raises ``ValueError`` when the source contains no function definition. + """ + func_tree = cst.parse_module(definition) + for stmt in func_tree.body: + if isinstance(stmt, cst.FunctionDef): + return stmt + raise ValueError("--definition must contain a function definition.") + + +def insert_imports(body: list[cst.BaseStatement], imports: list[cst.BaseStatement]) -> None: + """Insert import statements after the last existing import (in place).""" + import_insert_idx = 0 + for i, stmt in enumerate(body): + if isinstance(stmt, cst.SimpleStatementLine): + for item in stmt.body: + if isinstance(item, (cst.Import, cst.ImportFrom)): + import_insert_idx = i + 1 + for stmt in reversed(imports): + body.insert(import_insert_idx, stmt) + + +def insert_before_assignments( + body: list[cst.BaseStatement], + stmts: list[cst.BaseStatement], + *, + target_names: set[str], + runnable_names: set[str] = frozenset(), + step_calls_of: str | None = None, +) -> None: + """Insert statements before the earliest anchoring statement (in place). + + Anchors are top-level assignments (plain or annotated) whose target is in + *target_names*, or whose Call value resolves (via ``resolve_runnable_name``) + to a name in *runnable_names*. When *step_calls_of* is set, standalone + ``.step(...)`` statements also anchor — this covers aliased + entry points that never appear as assignments. Appends at the end when no + anchor is found. + """ + insert_idx = len(body) + for i, stmt in enumerate(body): + if isinstance(stmt, cst.SimpleStatementLine): + for item in stmt.body: + if isinstance(item, cst.Assign): + if runnable_names and isinstance(item.value, cst.Call): + if resolve_runnable_name(item.value) in runnable_names: + insert_idx = min(insert_idx, i) + for t in item.targets: + if isinstance(t.target, cst.Name) and t.target.value in target_names: + insert_idx = min(insert_idx, i) + elif isinstance(item, cst.AnnAssign): + if isinstance(item.target, cst.Name): + if item.target.value in target_names: + insert_idx = min(insert_idx, i) + elif ( + runnable_names + and isinstance(item.value, cst.Call) + and resolve_runnable_name(item.value) in runnable_names + ): + insert_idx = min(insert_idx, i) + elif ( + step_calls_of is not None + and isinstance(item, cst.Expr) + and isinstance(item.value, cst.Call) + and is_step_call(item.value, step_calls_of) + ): + insert_idx = min(insert_idx, i) + for stmt in reversed(stmts): + body.insert(insert_idx, stmt) + + +def validate_tools_target( + tree: cst.Module | None, + entry_point: str, + step: str | None, + *, + operation: str, +) -> tuple[str, dict[str, cst.Call]]: + """Validate the add-tool/add-mcp target and return (target, assignments). + + With ``--step`` the entry point must be a Workflow and the step must exist; + without it the entry point must be an Agent assigned in the source. + """ + if tree is not None: + ep_type = resolve_entry_point_type(tree, entry_point) + if step: + if ep_type is not None and ep_type != "Workflow": + raise ValueError(f"--step requires a Workflow entry point, but '{entry_point}' is a {ep_type}.") + else: + if ep_type is not None and ep_type != "Agent": + raise ValueError(f"{operation} requires an Agent entry point, but '{entry_point}' is a {ep_type}.") + + target = step if step else entry_point + assignments = collect_assignments(tree) if tree else {} + + if tree is not None: + if step: + step_names = collect_step_names(tree, entry_point, assignments) + if ( + step not in step_names + and step not in assignments + and not is_bare_function_step(tree, entry_point, step, assignments) + ): + raise ValueError( + f"Workflow step '{step}' not found. " + "Use the step variable name from .step(...), not the runtime name." + ) + elif entry_point not in assignments: + raise ValueError( + f"Entry point variable '{entry_point}' not found in source. " + "Ensure timbal.yaml fqn matches the Agent/Workflow variable name." + ) + + return target, assignments + + +def merge_config_kwargs(call: cst.Call, config: dict) -> cst.Call: + """Merge *config* kwargs into an existing Call, preserving all other args. + + Existing kwargs named in *config* are dropped (overridden or removed); + new kwargs are appended, skipping ``None`` values (None = remove). + """ + args = [ + a for a in call.args + if not (isinstance(a.keyword, cst.Name) and a.keyword.value in config) + ] + for key, value in config.items(): + if value is not None: + args.append(cst.Arg(keyword=cst.Name(key), value=build_cst_value(value))) + return call.with_changes(args=args) + + def has_import(tree: cst.Module, module: str, name: str) -> bool: """Check if `from import ` already exists.""" for stmt in tree.body: diff --git a/python/timbal/codegen/transformers/add_edge.py b/python/timbal/codegen/transformers/add_edge.py index c374c33b..27afa219 100644 --- a/python/timbal/codegen/transformers/add_edge.py +++ b/python/timbal/codegen/transformers/add_edge.py @@ -4,10 +4,12 @@ from ..cli_utils import arg_input from ..cst_utils import ( + StepCallRewriter, collect_assignments, collect_chained_step_names, collect_step_names, has_import, + insert_imports, require_step, resolve_entry_point_type, ) @@ -59,7 +61,7 @@ def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = return EdgeAdder(entry_point, source, args.target, when_expr, assignments, step_names) -class EdgeAdder(cst.CSTTransformer): +class EdgeAdder(StepCallRewriter): """Add an ordering or conditional edge between two workflow steps.""" needs_reorder = True @@ -83,34 +85,6 @@ def __init__( self.assignments = assignments self.step_names = step_names or {} - def _is_step_call(self, call: cst.Call) -> bool: - return ( - isinstance(call.func, cst.Attribute) - and isinstance(call.func.value, cst.Name) - and call.func.value.value == self.entry_point - and call.func.attr.value == "step" - ) - - def _matches_target(self, call: cst.Call) -> bool: - """Check if a .step() call is the target step. - - Matches when self.target equals either the variable name or the - runtime name (the ``name=`` kwarg) of the step. - """ - if not call.args: - return False - first_arg = call.args[0].value - if isinstance(first_arg, cst.Name): - var_name = first_arg.value - # Match by variable name. - if var_name == self.target: - return True - # Match by runtime name (name= kwarg). - runtime_name = self.step_names.get(var_name) - if runtime_name is not None and runtime_name == self.target: - return True - return False - def _get_existing_depends_on(self, call: cst.Call) -> list[str]: """Extract existing depends_on list from a .step() call.""" for arg in call.args[1:]: @@ -123,34 +97,10 @@ def _get_existing_depends_on(self, call: cst.Call) -> list[str]: return deps return [] - def leave_Expr(self, original_node: cst.Expr, updated_node: cst.Expr) -> cst.Expr: - """Update the target's .step() call to add the edge.""" - call = updated_node.value - if not isinstance(call, cst.Call): - return updated_node - if not self._is_step_call(call) or not self._matches_target(call): - return updated_node - - step_call_code = self._build_step_call_code(call) - parsed = cst.parse_module(step_call_code + "\n") - for stmt in parsed.body: - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): - return updated_node.with_changes(value=item.value) - return updated_node - def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: if self.when_expr and not has_import(original_node, "timbal.state", "get_run_context"): body = list(updated_node.body) - import_stmt = cst.parse_statement("from timbal.state import get_run_context\n") - import_insert_idx = 0 - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, (cst.Import, cst.ImportFrom)): - import_insert_idx = i + 1 - body.insert(import_insert_idx, import_stmt) + insert_imports(body, [cst.parse_statement("from timbal.state import get_run_context\n")]) return updated_node.with_changes(body=body) return updated_node diff --git a/python/timbal/codegen/transformers/add_mcp.py b/python/timbal/codegen/transformers/add_mcp.py index 7573566b..9d64c633 100644 --- a/python/timbal/codegen/transformers/add_mcp.py +++ b/python/timbal/codegen/transformers/add_mcp.py @@ -14,12 +14,11 @@ from ..cli_utils import arg_input, parse_json_arg from ..cst_utils import ( - collect_assignments, - collect_step_names, has_import, - is_bare_function_step, - resolve_entry_point_type, + insert_before_assignments, + insert_imports, resolve_runnable_name, + validate_tools_target, ) _ENV_PLACEHOLDER = re.compile(r"\$\{(?P[A-Za-z_][A-Za-z0-9_]*)\}|\$(?P[A-Za-z_][A-Za-z0-9_]*)") @@ -166,35 +165,7 @@ def _servers_from_args(args: argparse.Namespace) -> list[tuple[str, dict]]: def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = None) -> cst.CSTTransformer: step = getattr(args, "step", None) - if tree is not None: - ep_type = resolve_entry_point_type(tree, entry_point) - if step: - if ep_type is not None and ep_type != "Workflow": - raise ValueError(f"--step requires a Workflow entry point, but '{entry_point}' is a {ep_type}.") - else: - if ep_type is not None and ep_type != "Agent": - raise ValueError(f"add-mcp requires an Agent entry point, but '{entry_point}' is a {ep_type}.") - - target = step if step else entry_point - assignments = collect_assignments(tree) if tree else {} - - if tree is not None: - if step: - step_names = collect_step_names(tree, entry_point, assignments) - if ( - step not in step_names - and step not in assignments - and not is_bare_function_step(tree, entry_point, step, assignments) - ): - raise ValueError( - f"Workflow step '{step}' not found. " - "Use the step variable name from .step(...), not the runtime name." - ) - elif entry_point not in assignments: - raise ValueError( - f"Entry point variable '{entry_point}' not found in source. " - "Ensure timbal.yaml fqn matches the Agent/Workflow variable name." - ) + target, assignments = validate_tools_target(tree, entry_point, step, operation="add-mcp") servers = _servers_from_args(args) @@ -399,42 +370,12 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c return updated_node body = list(updated_node.body) - - if imports_to_add: - import_insert_idx = 0 - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, (cst.Import, cst.ImportFrom)): - import_insert_idx = i + 1 - for stmt in reversed(imports_to_add): - body.insert(import_insert_idx, stmt) - - if stmts_to_add: - insert_idx = len(body) - runtime_names = {rn for _, rn, _ in self.servers} - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Assign) and isinstance(item.value, cst.Call): - if resolve_runnable_name(item.value) in runtime_names: - insert_idx = min(insert_idx, i) - for t in item.targets: - if isinstance(t.target, cst.Name) and t.target.value == self.target: - insert_idx = min(insert_idx, i) - elif ( - isinstance(item, cst.AnnAssign) - and isinstance(item.target, cst.Name) - and isinstance(item.value, cst.Call) - ): - if ( - resolve_runnable_name(item.value) in runtime_names - or item.target.value == self.target - ): - insert_idx = min(insert_idx, i) - for stmt in reversed(stmts_to_add): - body.insert(insert_idx, stmt) - + insert_imports(body, imports_to_add) + insert_before_assignments( + body, stmts_to_add, + target_names={self.target}, + runnable_names={rn for _, rn, _ in self.servers}, + ) return updated_node.with_changes(body=body) def _add_to_tools(self, call: cst.Call) -> cst.Call: diff --git a/python/timbal/codegen/transformers/add_step.py b/python/timbal/codegen/transformers/add_step.py index 3b712352..bcd9a951 100644 --- a/python/timbal/codegen/transformers/add_step.py +++ b/python/timbal/codegen/transformers/add_step.py @@ -8,6 +8,10 @@ collect_assignments, has_import, has_step_expr, + insert_before_assignments, + insert_imports, + is_step_call, + parse_function_def, resolve_entry_point_type, resolve_runnable_name, ) @@ -234,14 +238,7 @@ def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = _reject_non_name_config(config, "Custom") if not args.definition: raise ValueError("--definition is required for Custom steps.") - func_tree = cst.parse_module(args.definition) - func_def = None - for stmt in func_tree.body: - if isinstance(stmt, cst.FunctionDef): - func_def = stmt - break - if func_def is None: - raise ValueError("--definition must contain a function definition.") + func_def = parse_function_def(args.definition) func_name = func_def.name.value runtime_name = args.step_name or config.get("name") or func_name var_name = runtime_name @@ -400,7 +397,7 @@ def leave_Expr(self, original_node: cst.Expr, updated_node: cst.Expr) -> cst.Exp call = updated_node.value if not isinstance(call, cst.Call): return updated_node - if not self._is_step_call(call): + if not is_step_call(call, self.entry_point): return updated_node # Check if first arg resolves to our step name. @@ -452,39 +449,14 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c stmts_to_add.append(assign_stmt) body = list(updated_node.body) - - # Insert imports after the last existing import. - if imports_to_add: - import_insert_idx = 0 - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, (cst.Import, cst.ImportFrom)): - import_insert_idx = i + 1 - for stmt in reversed(imports_to_add): - body.insert(import_insert_idx, stmt) - + insert_imports(body, imports_to_add) # Insert definitions/assignments before the entry point (plain or # annotated assignment), or before its first standalone .step() call # (aliased entry points), so the new step variable precedes the # ``.step()`` statement that references it. - if stmts_to_add: - insert_idx = len(body) - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Assign): - for t in item.targets: - if isinstance(t.target, cst.Name) and t.target.value == self.entry_point: - insert_idx = min(insert_idx, i) - elif isinstance(item, cst.AnnAssign): - if isinstance(item.target, cst.Name) and item.target.value == self.entry_point: - insert_idx = min(insert_idx, i) - elif isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): - if self._is_step_call(item.value): - insert_idx = min(insert_idx, i) - for stmt in reversed(stmts_to_add): - body.insert(insert_idx, stmt) + insert_before_assignments( + body, stmts_to_add, target_names={self.entry_point}, step_calls_of=self.entry_point, + ) # --- Append .step() call --- if not self._step_call_updated: @@ -499,7 +471,7 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c if isinstance(stmt, cst.SimpleStatementLine): for item in stmt.body: if isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): - if self._is_step_call(item.value): + if is_step_call(item.value, self.entry_point): step_insert_idx = i + 1 if isinstance(item, cst.Assign): for t in item.targets: @@ -519,15 +491,6 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c # -- Helpers ------------------------------------------------------------ - def _is_step_call(self, call: cst.Call) -> bool: - """Check if a Call node is entry_point.step(...).""" - return ( - isinstance(call.func, cst.Attribute) - and isinstance(call.func.value, cst.Name) - and call.func.value.value == self.entry_point - and call.func.attr.value == "step" - ) - def _resolve_step_name(self, element: cst.BaseExpression) -> str | None: """Resolve the name of a step from the first arg of .step().""" if isinstance(element, cst.Name): diff --git a/python/timbal/codegen/transformers/add_tool.py b/python/timbal/codegen/transformers/add_tool.py index 36d0be02..0dbfa3c9 100644 --- a/python/timbal/codegen/transformers/add_tool.py +++ b/python/timbal/codegen/transformers/add_tool.py @@ -5,12 +5,12 @@ from ..cli_utils import arg_input, parse_json_arg from ..cst_utils import ( build_cst_value, - collect_assignments, - collect_step_names, has_import, - is_bare_function_step, - resolve_entry_point_type, + insert_before_assignments, + insert_imports, + parse_function_def, resolve_runnable_name, + validate_tools_target, ) from ..tool_discovery import get_framework_tool_names, get_framework_tools, validate_tool_config @@ -56,35 +56,7 @@ def register(subparsers: argparse._SubParsersAction) -> None: def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = None) -> cst.CSTTransformer: step = getattr(args, "step", None) - if tree is not None: - ep_type = resolve_entry_point_type(tree, entry_point) - if step: - if ep_type is not None and ep_type != "Workflow": - raise ValueError(f"--step requires a Workflow entry point, but '{entry_point}' is a {ep_type}.") - else: - if ep_type is not None and ep_type != "Agent": - raise ValueError(f"add-tool requires an Agent entry point, but '{entry_point}' is a {ep_type}.") - - target = step if step else entry_point - assignments = collect_assignments(tree) if tree else {} - - if tree is not None: - if step: - step_names = collect_step_names(tree, entry_point, assignments) - if ( - step not in step_names - and step not in assignments - and not is_bare_function_step(tree, entry_point, step, assignments) - ): - raise ValueError( - f"Workflow step '{step}' not found. " - "Use the step variable name from .step(...), not the runtime name." - ) - elif entry_point not in assignments: - raise ValueError( - f"Entry point variable '{entry_point}' not found in source. " - "Ensure timbal.yaml fqn matches the Agent/Workflow variable name." - ) + target, assignments = validate_tools_target(tree, entry_point, step, operation="add-tool") tool_type = args.tool_type valid_types = [*get_framework_tools().keys(), "Custom"] @@ -119,15 +91,7 @@ def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = if tool_type == "Custom": if not args.definition: raise ValueError("--definition is required for Custom tools.") - func_tree = cst.parse_module(args.definition) - func_def = None - for stmt in func_tree.body: - if isinstance(stmt, cst.FunctionDef): - func_def = stmt - break - if func_def is None: - raise ValueError("--definition must contain a function definition.") - func_name = func_def.name.value + func_name = parse_function_def(args.definition).name.value var_name = args.tool_name if args.tool_name is not None else f"{func_name}_tool" runtime_name = args.tool_name if args.tool_name is not None else func_name return ToolAdder( @@ -280,44 +244,12 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c return updated_node body = list(updated_node.body) - - # Insert imports after the last existing import. - if imports_to_add: - import_insert_idx = 0 - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, (cst.Import, cst.ImportFrom)): - import_insert_idx = i + 1 - for stmt in reversed(imports_to_add): - body.insert(import_insert_idx, stmt) - - # Insert statements before the earliest relevant assignment (plain or - # annotated): either a tool wrapper assignment or the target - # (entry point / step). - if stmts_to_add: - insert_idx = len(body) - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Assign) and isinstance(item.value, cst.Call): - resolved = resolve_runnable_name(item.value) - if resolved == self.runtime_name: - insert_idx = min(insert_idx, i) - for t in item.targets: - if isinstance(t.target, cst.Name) and t.target.value == self.target: - insert_idx = min(insert_idx, i) - elif ( - isinstance(item, cst.AnnAssign) - and isinstance(item.target, cst.Name) - and isinstance(item.value, cst.Call) - ): - resolved = resolve_runnable_name(item.value) - if resolved == self.runtime_name or item.target.value == self.target: - insert_idx = min(insert_idx, i) - for stmt in reversed(stmts_to_add): - body.insert(insert_idx, stmt) - + insert_imports(body, imports_to_add) + # Insert statements before the earliest relevant assignment: + # either a tool wrapper assignment or the target (entry point / step). + insert_before_assignments( + body, stmts_to_add, target_names={self.target}, runnable_names={self.runtime_name}, + ) return updated_node.with_changes(body=body) # -- Helpers ------------------------------------------------------------ diff --git a/python/timbal/codegen/transformers/remove_edge.py b/python/timbal/codegen/transformers/remove_edge.py index ffabb13f..9f0977a5 100644 --- a/python/timbal/codegen/transformers/remove_edge.py +++ b/python/timbal/codegen/transformers/remove_edge.py @@ -3,6 +3,7 @@ import libcst as cst from ..cst_utils import ( + StepCallRewriter, collect_assignments, collect_chained_step_names, collect_step_names, @@ -46,7 +47,7 @@ def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = return EdgeRemover(entry_point, args.source, args.target, assignments, step_names) -class EdgeRemover(cst.CSTTransformer): +class EdgeRemover(StepCallRewriter): """Remove an edge between two workflow steps by modifying the target's .step() call.""" # Removing an edge that is already absent is an idempotent success. @@ -68,50 +69,11 @@ def __init__( # Resolve source to runtime name for matching against depends_on and step_span. self._resolved_source = self.step_names.get(source, source) - def _is_step_call(self, call: cst.Call) -> bool: - return ( - isinstance(call.func, cst.Attribute) - and isinstance(call.func.value, cst.Name) - and call.func.value.value == self.entry_point - and call.func.attr.value == "step" - ) - - def _matches_target(self, call: cst.Call) -> bool: - """Match by variable name or runtime name.""" - if not call.args: - return False - first_arg = call.args[0].value - if isinstance(first_arg, cst.Name): - var_name = first_arg.value - if var_name == self.target: - return True - runtime_name = self.step_names.get(var_name) - if runtime_name is not None and runtime_name == self.target: - return True - return False - def _references_source(self, arg: cst.Arg) -> bool: """Check if an argument's value references the source step via step_span.""" code = cst.parse_module("").code_for_node(arg.value) return f'step_span("{self._resolved_source}")' in code - def leave_Expr(self, original_node: cst.Expr, updated_node: cst.Expr) -> cst.Expr: - """Remove edge references from the target's .step() call.""" - call = updated_node.value - if not isinstance(call, cst.Call): - return updated_node - if not self._is_step_call(call) or not self._matches_target(call): - return updated_node - - step_call_code = self._build_step_call_code(call) - parsed = cst.parse_module(step_call_code + "\n") - for stmt in parsed.body: - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): - return updated_node.with_changes(value=item.value) - return updated_node - def _build_step_call_code(self, existing_call: cst.Call) -> str: """Rebuild the .step() call with source references removed.""" first_arg = existing_call.args[0] diff --git a/python/timbal/codegen/transformers/remove_step.py b/python/timbal/codegen/transformers/remove_step.py index c6857141..22fc56da 100644 --- a/python/timbal/codegen/transformers/remove_step.py +++ b/python/timbal/codegen/transformers/remove_step.py @@ -5,6 +5,7 @@ from ..cst_utils import ( collect_assignments, has_step_expr, + is_step_call, resolve_entry_point_type, resolve_runnable_name, ) @@ -61,7 +62,7 @@ def leave_SimpleStatementLine( for item in updated_node.body: if isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): call = item.value - if not self._is_standalone_step_call(call): + if not is_step_call(call, self.entry_point): continue # Remove the target step entirely. if self._matches_step_name(call): @@ -115,15 +116,6 @@ def _is_dot_step(self, call: cst.Call) -> bool: """Check if a Call node is .step(...).""" return isinstance(call.func, cst.Attribute) and call.func.attr.value == "step" - def _is_standalone_step_call(self, call: cst.Call) -> bool: - """Check if a Call node is entry_point.step(...) as a standalone expression.""" - return ( - isinstance(call.func, cst.Attribute) - and isinstance(call.func.value, cst.Name) - and call.func.value.value == self.entry_point - and call.func.attr.value == "step" - ) - def _matches_step_name(self, call: cst.Call) -> bool: """Check if the first argument of .step() resolves to the target step name.""" if not call.args: diff --git a/python/timbal/codegen/transformers/set_config.py b/python/timbal/codegen/transformers/set_config.py index a54eafb9..6a0ce5a3 100644 --- a/python/timbal/codegen/transformers/set_config.py +++ b/python/timbal/codegen/transformers/set_config.py @@ -4,9 +4,12 @@ from ..cli_utils import arg_input, parse_json_arg from ..cst_utils import ( + assignment_resolves_to, build_cst_value, collect_assignments, + insert_before_assignments, is_bare_function_step, + merge_config_kwargs, resolve_entry_point_type, resolve_runnable_name, wrap_bare_function_step, @@ -334,23 +337,8 @@ def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> c return updated_node body = list(updated_node.body) - # Insert before the entry point assignment (plain or annotated). - insert_idx = len(body) - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Assign): - for t in item.targets: - if isinstance(t.target, cst.Name) and t.target.value == self.entry_point: - insert_idx = min(insert_idx, i) - elif isinstance(item, cst.AnnAssign): - if isinstance(item.target, cst.Name) and item.target.value == self.entry_point: - insert_idx = min(insert_idx, i) - - for stmt in reversed(stmts_to_add): - body.insert(insert_idx, stmt) - + insert_before_assignments(body, stmts_to_add, target_names={self.entry_point}) return updated_node.with_changes(body=body) def _migrate_inline(self, call: cst.Call) -> cst.Call: @@ -374,16 +362,7 @@ def _migrate_inline(self, call: cst.Call) -> cst.Call: def _build_configured_call(self, existing_call: cst.Call) -> cst.Call: """Merge new config kwargs into an existing Call, preserving all other args.""" - # Keep existing args except those being overridden or removed. - args = [ - a for a in existing_call.args - if not (isinstance(a.keyword, cst.Name) and a.keyword.value in self.config) - ] - # Append new/updated config kwargs (None = remove, already dropped above). - for key, value in self.config.items(): - if value is not None: - args.append(cst.Arg(keyword=cst.Name(key), value=build_cst_value(value))) - return existing_call.with_changes(args=args) + return merge_config_kwargs(existing_call, self.config) class StepConstructorConfigSetter(cst.CSTTransformer): @@ -414,47 +393,22 @@ def __init__( # -- Constructor kwargs update --------------------------------------------- def leave_Assign(self, original_node: cst.Assign, updated_node: cst.Assign) -> cst.Assign: - for target in updated_node.targets: - if not isinstance(target.target, cst.Name): - continue - target_name = target.target.value - if target_name == self.entry_point: - continue - if isinstance(updated_node.value, cst.Call): - resolved = resolve_runnable_name(updated_node.value) - if resolved == self.step_name: - self.matched = True - return updated_node.with_changes( - value=self._merge_config_into_call(updated_node.value), - ) + if assignment_resolves_to(updated_node, self.entry_point, self.step_name): + self.matched = True + return updated_node.with_changes( + value=merge_config_kwargs(updated_node.value, self.config), + ) return updated_node def leave_AnnAssign(self, original_node: cst.AnnAssign, updated_node: cst.AnnAssign) -> cst.AnnAssign: # noqa: ARG002 """Handle annotated step variables, e.g. ``agent_a: Agent = Agent(...)``.""" - if ( - not isinstance(updated_node.target, cst.Name) - or updated_node.target.value == self.entry_point - or not isinstance(updated_node.value, cst.Call) - ): - return updated_node - if resolve_runnable_name(updated_node.value) == self.step_name: + if assignment_resolves_to(updated_node, self.entry_point, self.step_name): self.matched = True return updated_node.with_changes( - value=self._merge_config_into_call(updated_node.value), + value=merge_config_kwargs(updated_node.value, self.config), ) return updated_node - def _merge_config_into_call(self, call: cst.Call) -> cst.Call: - """Drop overridden kwargs and append the new config values.""" - args = [ - a for a in call.args - if not (isinstance(a.keyword, cst.Name) and a.keyword.value in self.config) - ] - for key, value in self.config.items(): - if value is not None: - args.append(cst.Arg(keyword=cst.Name(key), value=build_cst_value(value))) - return call.with_changes(args=args) - # -- Context tracking for depends_on / step_span string replacement -------- def visit_Arg(self, node: cst.Arg) -> bool: diff --git a/python/timbal/codegen/transformers/set_param.py b/python/timbal/codegen/transformers/set_param.py index 73bedfdf..30591aa9 100644 --- a/python/timbal/codegen/transformers/set_param.py +++ b/python/timbal/codegen/transformers/set_param.py @@ -6,10 +6,12 @@ from ..cli_utils import arg_input, parse_json_arg from ..cst_utils import ( + StepCallRewriter, collect_assignments, collect_chained_step_names, collect_step_names, has_import, + insert_imports, require_step, resolve_entry_point_type, resolve_runnable_name, @@ -172,7 +174,7 @@ def run(entry_point: str, args: argparse.Namespace, *, tree: cst.Module | None = raise ValueError(f"Unknown param type: {param_type}") -class ParamSetter(cst.CSTTransformer): +class ParamSetter(StepCallRewriter): """Set a parameter on a workflow step's .step() call.""" def __init__( @@ -197,18 +199,11 @@ def __init__( self.assignments = assignments or {} self.step_names = step_names or {} self.needs_reorder = param_type == "map" - # True once the target .step() call was found — an unchanged file is - # then an idempotent save (param already set to this value). + # Set to True by StepCallRewriter once the target .step() call was + # found — an unchanged file is then an idempotent save (param already + # set to this value). self.matched = False - def _is_step_call(self, call: cst.Call) -> bool: - return ( - isinstance(call.func, cst.Attribute) - and isinstance(call.func.value, cst.Name) - and call.func.value.value == self.entry_point - and call.func.attr.value == "step" - ) - def _source_uses_attribute_access(self) -> bool: """Whether the source step's output is accessed via attributes (not subscript). @@ -222,50 +217,10 @@ def _source_uses_attribute_access(self) -> bool: return call.func.value == "Agent" return False - def _matches_target(self, call: cst.Call) -> bool: - """Match by variable name or runtime name (mirrors add-edge).""" - if not call.args: - return False - first_arg = call.args[0].value - if isinstance(first_arg, cst.Name): - var_name = first_arg.value - if var_name == self.target: - return True - runtime_name = self.step_names.get(var_name) - if runtime_name is None and var_name in self.assignments: - runtime_name = resolve_runnable_name(self.assignments[var_name]) - if runtime_name is not None and runtime_name == self.target: - return True - return False - - def leave_Expr(self, original_node: cst.Expr, updated_node: cst.Expr) -> cst.Expr: - call = updated_node.value - if not isinstance(call, cst.Call): - return updated_node - if not self._is_step_call(call) or not self._matches_target(call): - return updated_node - - self.matched = True - step_call_code = self._build_step_call_code(call) - parsed = cst.parse_module(step_call_code + "\n") - for stmt in parsed.body: - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, cst.Expr) and isinstance(item.value, cst.Call): - return updated_node.with_changes(value=item.value) - return updated_node - def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: if self.param_type == "map" and not has_import(original_node, "timbal.state", "get_run_context"): body = list(updated_node.body) - import_stmt = cst.parse_statement("from timbal.state import get_run_context\n") - import_insert_idx = 0 - for i, stmt in enumerate(body): - if isinstance(stmt, cst.SimpleStatementLine): - for item in stmt.body: - if isinstance(item, (cst.Import, cst.ImportFrom)): - import_insert_idx = i + 1 - body.insert(import_insert_idx, import_stmt) + insert_imports(body, [cst.parse_statement("from timbal.state import get_run_context\n")]) return updated_node.with_changes(body=body) return updated_node diff --git a/python/timbal/codegen/transformers/set_position.py b/python/timbal/codegen/transformers/set_position.py index 735e9164..e2c27d7a 100644 --- a/python/timbal/codegen/transformers/set_position.py +++ b/python/timbal/codegen/transformers/set_position.py @@ -3,6 +3,7 @@ import libcst as cst from ..cst_utils import ( + assignment_resolves_to, build_cst_value, collect_assignments, collect_chained_step_names, @@ -10,7 +11,6 @@ is_bare_function_step, require_step, resolve_entry_point_type, - resolve_runnable_name, wrap_bare_function_step, ) @@ -158,31 +158,17 @@ def __init__(self, entry_point: str, step_name: str, position: dict, assignments self.matched = False # target step found — unchanged file means idempotent save def leave_Assign(self, original_node: cst.Assign, updated_node: cst.Assign) -> cst.Assign: - for target in updated_node.targets: - if not isinstance(target.target, cst.Name): - continue - target_name = target.target.value - if target_name == self.entry_point: - continue - if isinstance(updated_node.value, cst.Call): - resolved = resolve_runnable_name(updated_node.value) - if resolved == self.step_name: - self.matched = True - new_args = _merge_position_into_metadata(list(updated_node.value.args), self.position) - return updated_node.with_changes( - value=updated_node.value.with_changes(args=new_args), - ) + if assignment_resolves_to(updated_node, self.entry_point, self.step_name): + self.matched = True + new_args = _merge_position_into_metadata(list(updated_node.value.args), self.position) + return updated_node.with_changes( + value=updated_node.value.with_changes(args=new_args), + ) return updated_node def leave_AnnAssign(self, original_node: cst.AnnAssign, updated_node: cst.AnnAssign) -> cst.AnnAssign: # noqa: ARG002 """Handle annotated step variables, e.g. ``agent_a: Agent = Agent(...)``.""" - if ( - not isinstance(updated_node.target, cst.Name) - or updated_node.target.value == self.entry_point - or not isinstance(updated_node.value, cst.Call) - ): - return updated_node - if resolve_runnable_name(updated_node.value) == self.step_name: + if assignment_resolves_to(updated_node, self.entry_point, self.step_name): self.matched = True new_args = _merge_position_into_metadata(list(updated_node.value.args), self.position) return updated_node.with_changes( diff --git a/python/timbal/collectors/__init__.py b/python/timbal/collectors/__init__.py index 69813524..71370b97 100644 --- a/python/timbal/collectors/__init__.py +++ b/python/timbal/collectors/__init__.py @@ -1,29 +1,49 @@ # ruff: noqa: F401 +from typing import Any + from .registry import CollectorRegistry # Private registry instance _collector_registry = CollectorRegistry() -_collectors_loaded = False +_base_collectors_loaded = False +_sdk_collectors_loaded: dict[str, bool] = {"anthropic": False, "openai": False} + + +def _ensure_sdk_collector_for(event: Any) -> None: + """Import a provider-SDK collector only when a chunk from that SDK appears. + + A stream chunk's class lives in its SDK's module tree, so the module name + prefix identifies the provider without importing anything. This keeps an + anthropic-only process from ever importing openai (and vice versa) — the + collector modules import their SDK's types at module level. + """ + root = type(event).__module__.split(".", 1)[0] + if _sdk_collectors_loaded.get(root) is False: + _sdk_collectors_loaded[root] = True + if root == "anthropic": + from .impl import anthropic # noqa: F811 + else: + from .impl import openai # noqa: F811 -def _ensure_collectors_loaded(): - """Lazily import collector implementations on first use. +def _ensure_base_collectors_loaded(): + """Lazily import the SDK-free collector implementations on first use. - This avoids eagerly importing anthropic/openai SDKs at package import time. + NOTE: Do not register the default collector, this might lead to + unexpected behavior when detecting the correct collector type. """ - global _collectors_loaded - if _collectors_loaded: + global _base_collectors_loaded + if _base_collectors_loaded: return - _collectors_loaded = True - # Import all collector implementations to register them. - # NOTE: Do not register the default collector, this might lead to - # unexpected behavior when detecting the correct collector type. - from .impl import anthropic, openai, string, timbal # noqa: F811, E402 + _base_collectors_loaded = True + from .impl import string, timbal # noqa: F811, E402 + + _collector_registry.lazy_loader = _ensure_sdk_collector_for def get_collector_registry(): """Get the global collector registry instance.""" - _ensure_collectors_loaded() + _ensure_base_collectors_loaded() return _collector_registry diff --git a/python/timbal/collectors/impl/openai.py b/python/timbal/collectors/impl/openai.py index e7cfb1f2..cb9dd6eb 100644 --- a/python/timbal/collectors/impl/openai.py +++ b/python/timbal/collectors/impl/openai.py @@ -413,7 +413,7 @@ def result(self) -> Message: # Openai allows the use of custom IDs for tool calls. # We choose to generate our own random IDs for consistency and to make sure they don't collide # (they are not transparent with the algs being used) - tool_calls = [{**tc, "id": uuid7(as_type="str").replace("-", "")} for tc in self._tool_calls] + tool_calls = [{**tc, "id": uuid7(as_type="hex")} for tc in self._tool_calls] content.extend(tool_calls) return Message.validate({"role": "assistant", "content": content, "stop_reason": self._stop_reason}) diff --git a/python/timbal/collectors/registry.py b/python/timbal/collectors/registry.py index d1c126c6..7e78af42 100644 --- a/python/timbal/collectors/registry.py +++ b/python/timbal/collectors/registry.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from typing import Any from .base import BaseCollector @@ -6,18 +7,24 @@ class CollectorRegistry: """Registry for managing event collector types.""" - + def __init__(self): self._collector_types: list[type[BaseCollector]] = [] - + self.lazy_loader: Callable[[Any], None] | None = None + """Optional hook called with the event before dispatch — used to + lazily import provider-SDK collector implementations only when a + chunk from that SDK actually appears (see collectors.__init__).""" + def register(self, collector_type: type[BaseCollector]) -> None: """Register a new event collector type.""" self._collector_types.append(collector_type) - + def get_collector_type(self, event: Any) -> type[BaseCollector] | None: """Get the appropriate collector type for the given event.""" + if self.lazy_loader is not None: + self.lazy_loader(event) for collector_type in self._collector_types: if collector_type.can_handle(event): return collector_type - + return DefaultCollector diff --git a/python/timbal/core/agent.py b/python/timbal/core/agent.py index 958f605b..7cfbb791 100644 --- a/python/timbal/core/agent.py +++ b/python/timbal/core/agent.py @@ -20,7 +20,6 @@ BaseModel, ConfigDict, Field, - PrivateAttr, SecretStr, SkipValidation, ValidationError, @@ -36,7 +35,7 @@ from ..types.message import Message from ..types.run_status import RunStatus from ..utils import coerce_to_dict, dump -from .llm_router import _llm_router +from .llm import _llm_router from .memory_compaction import MemoryCompactor from .models import Model, get_context_window from .runnable import Runnable, RunnableLike @@ -209,14 +208,13 @@ class Agent(Runnable): api_key: SecretStr | None = None """Custom API key for the LLM API.""" - _llm: Tool = PrivateAttr() - _system_prompt_fn: Callable | None = PrivateAttr(default=None) - _system_prompt_fn_is_async: bool = PrivateAttr(default=False) - _system_prompt_skills: str | None = PrivateAttr(default=None) - def model_post_init(self, __context: Any) -> None: """Initialize agent after Pydantic model creation.""" super().model_post_init(__context) + # Plain instance attributes (not PrivateAttr) — read on every agent turn. + self._system_prompt_fn: Callable | None = None + self._system_prompt_fn_is_async: bool = False + self._system_prompt_skills: str | None = None self._path = self.name # Handle callable system_prompt @@ -746,7 +744,17 @@ async def _maybe_compact_memory(self, current_span: Any, *, prev_usage: dict | N ) should_compact = True elif prev_usage: - prev_input_tokens = sum(v for k, v in prev_usage.items() if ":input" in k and "token" in k) + # Anthropic's input_tokens excludes cached tokens: full context = + # input_tokens + cache_creation_input_tokens + cache_read_input_tokens + # (disjoint totals). Count the two cache totals explicitly — not via a + # loose "input" match, which would double-count the flattened per-TTL + # breakdown keys (e.g. ephemeral_5m_input_tokens). + prev_input_tokens = sum( + v + for k, v in prev_usage.items() + if (":input" in k and "token" in k) + or k.endswith((":cache_creation_input_tokens", ":cache_read_input_tokens")) + ) prev_output_tokens = sum(v for k, v in prev_usage.items() if ":output" in k and "token" in k) utilization = (prev_input_tokens + prev_output_tokens) / context_window should_compact = utilization >= self.memory_compaction_ratio @@ -866,7 +874,7 @@ def _build_unknown_tool_event(self, tool_call: ToolUseContent, tools: list[Tool] run_id=run_context.id if run_context is not None else "", parent_run_id=None, path=f"{self._path}.{tool_call.name}", - call_id=uuid7(as_type="str").replace("-", ""), + call_id=uuid7(as_type="hex"), parent_call_id=None, input=tool_call.input, status=RunStatus(code="error", reason="tool_not_found", message=message), @@ -878,67 +886,108 @@ def _build_unknown_tool_event(self, tool_call: ToolUseContent, tools: list[Tool] metadata={"tool_not_found": True, "requested_tool": tool_call.name}, ) + def _build_dispatch_failed_event(self, tool_call: ToolUseContent, e: Exception) -> OutputEvent: + """Build the synthetic OutputEvent fed back to the LLM when tool dispatch itself fails.""" + logger.exception( + "Unexpected error dispatching tool; feeding error back to LLM.", + agent_path=self._path, + tool=tool_call.name, + ) + run_context = get_run_context() + now = int(time.time() * 1000) + return OutputEvent( + run_id=run_context.id if run_context is not None else "", + parent_run_id=None, + path=f"{self._path}.{tool_call.name}", + call_id=uuid7(as_type="hex"), + parent_call_id=None, + input=tool_call.input, + status=RunStatus(code="error", reason="dispatch_failed", message=str(e)), + output=None, + error={ + "type": type(e).__name__, + "message": str(e), + "traceback": traceback.format_exc(), + }, + t0=now, + t1=now, + usage={}, + metadata={"dispatch_failed": True, "requested_tool": tool_call.name}, + ) + + def _resolve_tool_for_call(self, tools: list[Tool], tool_call: ToolUseContent) -> Tool | None: + """Find the tool for a call, logging when the LLM asked for an unknown one.""" + tool = next((t for t in tools if t.name == tool_call.name), None) + if tool is None: + logger.warning( + "LLM called unknown tool; feeding error back so it can self-correct.", + agent_path=self._path, + requested_tool=tool_call.name, + available_tools=sorted(t.name for t in tools), + ) + return tool + + @staticmethod + def _link_tool_call_span(event: Any, tool_call: ToolUseContent) -> None: + """Link the LLM tool_call id to the tool's span for memory resolution.""" + if event.type == "START": + tool_call_span = get_run_context()._trace[event.call_id] + tool_call_span.metadata["tool_call_id"] = tool_call.id + async def _multiplex_tools(self, tools: list[Tool], tool_calls: list[ToolUseContent]) -> AsyncGenerator[Any, None]: """Execute multiple tool calls concurrently and multiplex their events.""" + # Fast path: a single tool call needs no queue/task fan-in — iterate the + # tool directly, exactly like Workflow consumes its steps. This is the + # common case for most agent iterations. + if len(tool_calls) == 1: + tool_call = tool_calls[0] + tool = self._resolve_tool_for_call(tools, tool_call) + if tool is None: + yield tool_call, self._build_unknown_tool_event(tool_call, tools) + return + # A cancelled tool records 'interrupted' on its own span and swallows + # the CancelledError. In the queue/task path the agent still sees the + # cancellation at queue.get(); here we share the task with the tool, + # so detect the still-pending cancel request and re-raise before + # forwarding post-cancel events (keeps LLM-output salvage semantics). + current_task = asyncio.current_task() + try: + # Raw stream: the collector wrapper is only needed at the public + # API boundary; internal consumers forward events themselves. + async for event in tool._stream(**tool_call.input): + if current_task is not None and current_task.cancelling(): + raise asyncio.CancelledError + self._link_tool_call_span(event, tool_call) + yield tool_call, event + except (asyncio.CancelledError, GeneratorExit, InterruptError): + raise + except Exception as e: + yield tool_call, self._build_dispatch_failed_event(tool_call, e) + if current_task is not None and current_task.cancelling(): + raise asyncio.CancelledError + return + queue = asyncio.Queue() tasks = [] async def consume_tool(tool_call: ToolUseContent): try: - tool = next((t for t in tools if t.name == tool_call.name), None) + tool = self._resolve_tool_for_call(tools, tool_call) if tool is None: - logger.warning( - "LLM called unknown tool; feeding error back so it can self-correct.", - agent_path=self._path, - requested_tool=tool_call.name, - available_tools=sorted(t.name for t in tools), - ) - await queue.put((tool_call, self._build_unknown_tool_event(tool_call, tools))) + queue.put_nowait((tool_call, self._build_unknown_tool_event(tool_call, tools))) return - async for event in tool(**tool_call.input): - # Link tool call id to span for memory resolution - if event.type == "START": - tool_call_id = event.call_id - tool_call_span = get_run_context()._trace[tool_call_id] - tool_call_span.metadata["tool_call_id"] = tool_call.id - await queue.put((tool_call, event)) + async for event in tool._stream(**tool_call.input): + self._link_tool_call_span(event, tool_call) + queue.put_nowait((tool_call, event)) except (asyncio.CancelledError, GeneratorExit, InterruptError): raise except Exception as e: # Defensive: any unexpected error in dispatch must NOT swallow the # sentinel. Without this, a bug here would hang the agent forever # because the consumer loop waits on `remaining` to hit zero. - logger.exception( - "Unexpected error dispatching tool; feeding error back to LLM.", - agent_path=self._path, - tool=tool_call.name, - ) - run_context = get_run_context() - now = int(time.time() * 1000) - await queue.put(( - tool_call, - OutputEvent( - run_id=run_context.id if run_context is not None else "", - parent_run_id=None, - path=f"{self._path}.{tool_call.name}", - call_id=uuid7(as_type="str").replace("-", ""), - parent_call_id=None, - input=tool_call.input, - status=RunStatus(code="error", reason="dispatch_failed", message=str(e)), - output=None, - error={ - "type": type(e).__name__, - "message": str(e), - "traceback": traceback.format_exc(), - }, - t0=now, - t1=now, - usage={}, - metadata={"dispatch_failed": True, "requested_tool": tool_call.name}, - ), - )) + queue.put_nowait((tool_call, self._build_dispatch_failed_event(tool_call, e))) finally: - await queue.put((tool_call, None)) + queue.put_nowait((tool_call, None)) try: for tc in tool_calls: @@ -1096,14 +1145,16 @@ async def _process_tool_event(event: BaseEvent, tool_call_id: str, append_to_mes if command in commands: tool = commands[command] tool_input = {} - for i, field_name in enumerate(tool.params_model.model_fields.keys()): + # NOTE: do not name this loop variable `i` — it would shadow + # the agent-loop iteration counter in the enclosing scope. + for arg_idx, field_name in enumerate(tool.params_model.model_fields.keys()): # Params model preserves the ordering of the fields as they appear in the signature # We grab as many arguments as we can. If there are too few arguments, we'll let the tool params model validator give a better error - if i >= len(args): + if arg_idx >= len(args): break - tool_input[field_name] = args[i] + tool_input[field_name] = args[arg_idx] # Craft a fake tool_use so we can keep this interaction in the agent memory - tool_use_id = uuid7(as_type="str").replace("-", "") + tool_use_id = uuid7(as_type="hex") current_span._memory_dump.append( { "role": "assistant", @@ -1117,8 +1168,12 @@ async def _process_tool_event(event: BaseEvent, tool_call_id: str, append_to_mes ], } ) - # Run the tool - async for event in tool(**tool_input): + # Run the tool (raw stream — see _multiplex_tools; + # same pending-cancel detection as the fast path) + _cmd_task = asyncio.current_task() + async for event in tool._stream(**tool_input): + if _cmd_task is not None and _cmd_task.cancelling(): + raise asyncio.CancelledError await _process_tool_event(event, tool_use_id, append_to_messages=False) if isinstance(event, OutputEvent) and event.output is not None: if ( @@ -1192,7 +1247,7 @@ async def _process_tool_event(event: BaseEvent, tool_call_id: str, append_to_mes # Compaction rewrote memory out of lockstep with the dump; rebuild it. current_span._memory_dump = await dump(current_span.memory) - async for event in self._llm( + async for event in self._llm._stream( model=model, messages=current_span.memory, system_prompt=system_prompt, diff --git a/python/timbal/core/fallback_model.py b/python/timbal/core/fallback_model.py index 0cdd85b0..11e3f82c 100644 --- a/python/timbal/core/fallback_model.py +++ b/python/timbal/core/fallback_model.py @@ -5,16 +5,9 @@ from typing import Any import structlog -from anthropic import APIConnectionError as AnthropicAPIConnectionError -from anthropic import APIStatusError as AnthropicAPIStatusError -from anthropic import APITimeoutError as AnthropicAPITimeoutError -from anthropic import RateLimitError as AnthropicRateLimitError -from openai import APIConnectionError as OpenAIAPIConnectionError -from openai import APIStatusError as OpenAIAPIStatusError -from openai import APITimeoutError as OpenAIAPITimeoutError -from openai import RateLimitError as OpenAIRateLimitError from ..errors import FallbackExhausted +from .provider_errors import provider_error_classes logger = structlog.get_logger("timbal.core.fallback_model") @@ -131,13 +124,16 @@ def _should_fallback(self, exc: BaseException) -> bool: def is_retryable_provider_error(exc: BaseException) -> bool: - if isinstance(exc, (OpenAIRateLimitError, AnthropicRateLimitError)): + # Classes resolved lazily — this runs on an LLM error path, where the SDK + # that raised is already imported (see provider_errors). + err_cls = provider_error_classes() + if isinstance(exc, err_cls["rate_limit"]): return True - if isinstance(exc, (OpenAIAPITimeoutError, AnthropicAPITimeoutError)): + if isinstance(exc, err_cls["timeout"]): return True - if isinstance(exc, (OpenAIAPIConnectionError, AnthropicAPIConnectionError)): + if isinstance(exc, err_cls["connection"]): return True - if isinstance(exc, (OpenAIAPIStatusError, AnthropicAPIStatusError)): + if isinstance(exc, err_cls["status"]): status_code = getattr(exc, "status_code", None) if status_code is None: status_code = getattr(getattr(exc, "response", None), "status_code", None) diff --git a/python/timbal/core/llm/__init__.py b/python/timbal/core/llm/__init__.py new file mode 100644 index 00000000..5168a00c --- /dev/null +++ b/python/timbal/core/llm/__init__.py @@ -0,0 +1,48 @@ +""" +INTERNAL USE ONLY + +This package is intended for internal use and will be subject to frequent +changes as LLM providers constantly update their APIs and add new features. +The external APIs (Runnables, Agents, Workflows) will remain stable, but this +package will evolve to match provider changes. + +Do not rely on this package's interface in external code. + +Layout: + +- ``router`` — model-string dispatch (``_llm_router``) +- ``registry`` — static per-provider configuration (``_PROVIDERS``) +- ``clients`` — SDK client cache/resolution, platform proxy fallback, warmup +- ``retry`` — retry loop for transient provider failures + +One module per provider API shape (not per provider; Chat Completions alone +serves a dozen providers): + +- ``messages`` — Anthropic Messages API +- ``responses`` — OpenAI Responses API (openai, xai) +- ``chat_completions`` — OpenAI Chat Completions and compatible providers + +Each API module builds the request kwargs once per request and returns a +``(create_stream, context_label)`` pair consumed by the router's single retry +loop, so the split adds zero per-chunk overhead. + +Provider SDKs (openai, anthropic) are imported lazily — at client resolution +and error classification, never at package import. They account for ~460ms +(a third) of ``from timbal import Agent`` otherwise. +""" + +from .clients import _CLIENT_CACHE, _get_client, _resolve_client, warmup_llm_connection +from .registry import _PROVIDERS, _ProviderConfig +from .retry import _retry_on_error +from .router import _llm_router + +__all__ = [ + "_CLIENT_CACHE", + "_PROVIDERS", + "_ProviderConfig", + "_get_client", + "_llm_router", + "_resolve_client", + "_retry_on_error", + "warmup_llm_connection", +] diff --git a/python/timbal/core/llm/chat_completions.py b/python/timbal/core/llm/chat_completions.py new file mode 100644 index 00000000..f6c15059 --- /dev/null +++ b/python/timbal/core/llm/chat_completions.py @@ -0,0 +1,103 @@ +"""OpenAI Chat Completions request adapter. + +Serves every OpenAI-compatible provider (google, groq, togetherai, cerebras, +fireworks, moonshot, sambanova, xiaomi, byteplus, ...) plus openai/xai when +``TIMBAL_OPENAI_API`` is set to the legacy Chat Completions mode. Provider +quirks (reasoning content, stream options, text flattening) come from the +router's ``_ProviderConfig``. +""" + +from collections.abc import AsyncIterator, Callable +from typing import TYPE_CHECKING, Any + +from ...types.message import Message +from ...utils import transform_schema + +if TYPE_CHECKING: + from pydantic import BaseModel + + from ..runnable import Runnable + from .registry import _ProviderConfig + + +def prepare_chat_completions_request( + *, + provider: str, + config: "_ProviderConfig", + client: Any, + model_name: str, + request_headers: dict[str, str], + system_prompt: str | None, + messages: list[Message], + tools: "list[Runnable] | None", + max_tokens: int | None, + temperature: float | None, + output_model: "type[BaseModel] | None", + provider_params: dict[str, Any], +) -> tuple[Callable[[], AsyncIterator[Any]], str]: + """Build the Chat Completions kwargs and return (create_stream, context_label).""" + chat_completions_messages = [] + if system_prompt: + chat_completions_messages.append({"role": "system", "content": system_prompt}) + reasoning_as = "reasoning_content" if config.supports_chat_reasoning_content else "omit" + for message in messages: + chat_completions_message = message.to_openai_chat_completions_input(reasoning_as=reasoning_as) + # None = turn with no exportable payload (e.g. only server-side tool + # blocks after a cross-provider switch) — skip it. + if chat_completions_message is not None: + chat_completions_messages.append(chat_completions_message) + + # Some providers have incomplete OpenAI chat completions support. + # Flatten text-only content arrays to plain strings for compatibility. + if config.flatten_text_content: + for msg in chat_completions_messages: + content = msg.get("content") + if isinstance(content, list) and all( + isinstance(item, dict) and item.get("type") == "text" for item in content + ): + msg["content"] = "\n".join(item["text"] for item in content) + + chat_completions_kwargs: dict[str, Any] = { + "model": model_name, + "messages": chat_completions_messages, + "stream": True, + } + + if config.supports_stream_options: + chat_completions_kwargs["stream_options"] = {"include_usage": True} + + if tools: + chat_completions_tools = [tool.openai_chat_completions_schema for tool in tools] + if chat_completions_tools: + chat_completions_kwargs["tools"] = chat_completions_tools + + if max_tokens: + chat_completions_kwargs["max_completion_tokens"] = max_tokens + + if temperature is not None: + chat_completions_kwargs["temperature"] = temperature + + if output_model is not None: + chat_completions_kwargs["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": output_model.__name__, + "schema": transform_schema(output_model), + "strict": True, + }, + } + + # provider_params may carry extra tool defs; merge them with the client + # tools instead of letting dict.update clobber the generated list. + provider_params = dict(provider_params) + extra_tools = provider_params.pop("tools", None) + chat_completions_kwargs.update(provider_params) + if extra_tools: + chat_completions_kwargs["tools"] = [*chat_completions_kwargs.get("tools", []), *extra_tools] + + async def _create_stream(): + res = await client.chat.completions.create(extra_headers=request_headers, **chat_completions_kwargs) + async for chunk in res: + yield chunk + + return _create_stream, f"{provider} Chat Completions" diff --git a/python/timbal/core/llm/clients.py b/python/timbal/core/llm/clients.py new file mode 100644 index 00000000..cb739b0f --- /dev/null +++ b/python/timbal/core/llm/clients.py @@ -0,0 +1,131 @@ +"""SDK client cache and resolution (API keys, platform proxy fallback). + +Provider SDKs (openai, anthropic) are imported lazily — at client resolution, +never at module import. They account for ~460ms (a third) of +``from timbal import Agent`` otherwise. +""" + +import asyncio +import os +from types import SimpleNamespace +from typing import Any + +import structlog + +from ...errors import APIKeyNotFoundError +from .registry import _PROVIDERS, _ProviderConfig + +logger = structlog.get_logger("timbal.core.llm") + +# Module-level client cache keyed by (client_class, api_key, base_url, provider). +# Reusing clients preserves the underlying httpx connection pool, avoiding a +# fresh TCP+TLS handshake on every LLM call (~200-300ms saved per request). +# Per-request tracing headers (run_id, call_id) are passed via extra_headers +# on each individual .create() call instead. +# Concurrency-safe: all coroutines run on one thread in asyncio; the GIL +# ensures check-and-assign is atomic (no await between the if and the write). +# Values are AsyncOpenAI | AsyncAnthropic instances (SDKs imported lazily). +_CLIENT_CACHE: dict[tuple, Any] = {} + +# Shared httpx client for async file loading. Reused across LLM calls to +# preserve connection pools to file origins (CDNs, S3, etc.). Lazy-initialized +# on first use — never created if the conversation has no files. +# Loop-aware: httpx.AsyncClient is bound to the event loop at creation time. +# When the loop changes (e.g. between pytest-asyncio tests with per-function +# loop scope), the old client becomes unusable ("Event loop is closed"). +# We detect this and transparently recreate the client. +_FILE_CLIENT: Any = None +_FILE_CLIENT_LOOP: Any = None + + +def _get_file_client() -> Any: + global _FILE_CLIENT, _FILE_CLIENT_LOOP + loop = asyncio.get_running_loop() + if _FILE_CLIENT is not None and (_FILE_CLIENT.is_closed or _FILE_CLIENT_LOOP is not loop): + _FILE_CLIENT = None + if _FILE_CLIENT is None: + import httpx + + _FILE_CLIENT = httpx.AsyncClient(timeout=httpx.Timeout(30.0)) + _FILE_CLIENT_LOOP = loop + return _FILE_CLIENT + + +def _get_client(cls: type, api_key: str, base_url: str | None, provider: str) -> Any: + cache_key = (cls, api_key, base_url, provider) + if cache_key not in _CLIENT_CACHE: + kwargs: dict[str, Any] = {"api_key": api_key, "default_headers": {"x-provider": provider}} + if base_url: + kwargs["base_url"] = base_url + _CLIENT_CACHE[cache_key] = cls(**kwargs) + return _CLIENT_CACHE[cache_key] + + +def _resolve_client( + provider: str, + config: _ProviderConfig, + api_key: str | None, + base_url: str | None, + run_context: Any, +) -> tuple[Any, str | None]: + """Resolve API key, base URL, and return the appropriate SDK client. + + Returns: + (client, resolved_base_url) — base_url may have been updated for platform proxies. + """ + if not api_key: + api_key = os.getenv(config.env_key) + if not api_key: + if ( + config.supports_platform_proxy + and run_context.platform_config is not None + and run_context.platform_config.subject is not None + ): + api_key = run_context.platform_config.auth.header_value + base_url = ( + f"https://{run_context.platform_config.host}" + f"/orgs/{run_context.platform_config.subject.org_id}" + f"/proxies/{config.proxy_name}{config.proxy_suffix}" + ) + if not api_key: + raise APIKeyNotFoundError(f"{config.env_key} not found.") + + # Lazy SDK import: after the first call this is a sys.modules lookup (~1µs). + if config.client_type == "anthropic": + from anthropic import AsyncAnthropic + + return _get_client(AsyncAnthropic, api_key, base_url, "anthropic"), base_url + from openai import AsyncOpenAI + + return _get_client(AsyncOpenAI, api_key, base_url or config.default_base_url, provider), base_url + + +async def warmup_llm_connection(model: str) -> None: + """Pre-establish the provider's HTTPS connection pool for ``model``. + + The first LLM call of a process pays TCP+TLS(+HTTP/2) setup before any + token arrives — measured ~1.3s extra TTFT cold vs warm against + api.openai.com. This issues one lightweight authenticated GET + (``/models``) through the same cached SDK client the real calls will use, + so the pool is hot by the time the first request fires. + + Best-effort and side-effect free: any failure (no API key, unsupported + endpoint on OpenAI-compatible providers, timeout) is logged at DEBUG and + ignored. Callers fire-and-forget (e.g. voice sessions at startup). + """ + try: + provider, _ = model.split("/", 1) + except ValueError: + return + config = _PROVIDERS.get(provider) + if config is None: + return + try: + # _resolve_client only reads run_context.platform_config; don't create + # a real RunContext here (warmup must not touch tracing state). + ctx = SimpleNamespace(platform_config=None) + client, _ = _resolve_client(provider, config, None, None, ctx) + await asyncio.wait_for(client.models.list(), timeout=5.0) + logger.debug("llm_connection_warmed", model=model) + except Exception as e: + logger.debug("llm_warmup_skipped", model=model, error=str(e)) diff --git a/python/timbal/core/llm/messages.py b/python/timbal/core/llm/messages.py new file mode 100644 index 00000000..97b4b9c9 --- /dev/null +++ b/python/timbal/core/llm/messages.py @@ -0,0 +1,77 @@ +"""Anthropic Messages API request adapter (anthropic).""" + +from collections.abc import AsyncIterator, Callable +from typing import TYPE_CHECKING, Any + +from ...types.message import Message +from ...utils import transform_schema + +if TYPE_CHECKING: + from pydantic import BaseModel + + from ..runnable import Runnable + + +def prepare_messages_request( + *, + client: Any, + model_name: str, + request_headers: dict[str, str], + system_prompt: str | None, + messages: list[Message], + tools: "list[Runnable] | None", + max_tokens: int | None, + temperature: float | None, + output_model: "type[BaseModel] | None", + provider_params: dict[str, Any], +) -> tuple[Callable[[], AsyncIterator[Any]], str]: + """Build the Messages API kwargs and return (create_stream, context_label).""" + anthropic_kwargs = { + "model": model_name, + "messages": [message.to_anthropic_input() for message in messages], + "max_tokens": max_tokens, + "stream": True, + } + + if system_prompt: + anthropic_kwargs["system"] = system_prompt + + if tools: + anthropic_tools = [tool.anthropic_schema for tool in tools] + if anthropic_tools: + anthropic_kwargs["tools"] = anthropic_tools + + if temperature is not None: + anthropic_kwargs["temperature"] = temperature + + # Server-side automatic prompt caching (anthropic SDK >= 0.83): the API + # places a breakpoint on the last cacheable block and advances it as the + # conversation grows. Cache reads cost 0.1x input; prefixes below the + # model's minimum (1024-4096 tokens) are simply not cached, so this is + # safe to default on. Opt out with model_params={"cache_control": None}. + anthropic_kwargs["cache_control"] = {"type": "ephemeral"} + + # provider_params may carry provider-native (server-side) tool defs, e.g. + # {"type": "web_search_20250305", ...}. Merge them with the client tools + # instead of letting dict.update clobber the generated list. + provider_params = dict(provider_params) + extra_tools = provider_params.pop("tools", None) + anthropic_kwargs.update(provider_params) + if extra_tools: + anthropic_kwargs["tools"] = [*anthropic_kwargs.get("tools", []), *extra_tools] + if anthropic_kwargs["cache_control"] is None: + del anthropic_kwargs["cache_control"] + + async def _create_stream(): + if output_model is not None: + anthropic_kwargs["output_config"] = { + "format": { + "type": "json_schema", + "schema": transform_schema(output_model), + } + } + res = await client.messages.create(extra_headers=request_headers, **anthropic_kwargs) + async for chunk in res: + yield chunk + + return _create_stream, "Anthropic" diff --git a/python/timbal/core/llm/registry.py b/python/timbal/core/llm/registry.py new file mode 100644 index 00000000..c935f312 --- /dev/null +++ b/python/timbal/core/llm/registry.py @@ -0,0 +1,118 @@ +"""Provider registry — static per-provider configuration. + +Adding an OpenAI-compatible provider is one ``_PROVIDERS`` entry here; +adding a new API shape is a new module (see package docstring). +""" + +import os +from dataclasses import dataclass +from typing import Literal + +import structlog + +logger = structlog.get_logger("timbal.core.llm") + +TIMBAL_OPENAI_API = os.getenv("TIMBAL_OPENAI_API", "responses") +if TIMBAL_OPENAI_API != "responses": + logger.warning( + "Using legacy Chat Completions API. OpenAI is transitioning to the new Responses API, " + "which should be preferred for all new development. Set TIMBAL_OPENAI_API=responses to switch." + ) + + +@dataclass(frozen=True, slots=True) +class _ProviderConfig: + """Static configuration for a single LLM provider.""" + + env_key: str + """Environment variable name for the API key (e.g. ``OPENAI_API_KEY``).""" + + default_base_url: str | None = None + """Default API base URL. ``None`` uses the SDK default.""" + + proxy_name: str = "openai-completions" + """Platform proxy path segment (e.g. ``openai-responses``, ``anthropic``).""" + + proxy_suffix: str = "/v1" + """Appended to the proxy URL. Anthropic uses ``""``.""" + + client_type: Literal["openai", "anthropic"] = "openai" + """Which SDK client to create.""" + + flatten_text_content: bool = False + """Flatten text-only content arrays to plain strings for providers with incomplete Chat Completions support.""" + + supports_stream_options: bool = True + """Whether the provider supports ``stream_options`` in Chat Completions.""" + + supports_platform_proxy: bool = True + """If False, never fall back to the Timbal platform proxy — require the provider API key.""" + + supports_chat_reasoning_content: bool = False + """If True, serialize ThinkingContent as top-level ``reasoning_content`` (Moonshot/Fireworks-style). + + Otherwise thinking is omitted from outbound chat-completions messages (Vercel/LiteLLM default). + """ + + +_PROVIDERS: dict[str, _ProviderConfig] = { + "openai": _ProviderConfig( + env_key="OPENAI_API_KEY", + proxy_name="openai-responses" if TIMBAL_OPENAI_API == "responses" else "openai-completions", + ), + "anthropic": _ProviderConfig( + env_key="ANTHROPIC_API_KEY", + proxy_name="anthropic", + proxy_suffix="", + client_type="anthropic", + ), + "google": _ProviderConfig( + env_key="GEMINI_API_KEY", + default_base_url="https://generativelanguage.googleapis.com/v1beta/openai/", + ), + "togetherai": _ProviderConfig( + env_key="TOGETHER_API_KEY", + default_base_url="https://api.together.xyz/v1/", + supports_chat_reasoning_content=True, + ), + "xai": _ProviderConfig( + env_key="XAI_API_KEY", + default_base_url="https://api.x.ai/v1", + proxy_name="openai-responses", + ), + "groq": _ProviderConfig( + env_key="GROQ_API_KEY", + default_base_url="https://api.groq.com/openai/v1", + ), + "fireworks": _ProviderConfig( + env_key="FIREWORKS_API_KEY", + default_base_url="https://api.fireworks.ai/inference/v1", + supports_chat_reasoning_content=True, + ), + "byteplus": _ProviderConfig( + env_key="BYTEPLUS_API_KEY", + default_base_url="https://ark.ap-southeast.bytepluses.com/api/v3", + supports_chat_reasoning_content=True, + ), + "xiaomi": _ProviderConfig( + env_key="XIAOMI_API_KEY", + default_base_url="https://api.xiaomimimo.com/v1", + flatten_text_content=True, + supports_stream_options=False, + ), + "cerebras": _ProviderConfig( + env_key="CEREBRAS_API_KEY", + default_base_url="https://api.cerebras.ai/v1", + ), + "moonshot": _ProviderConfig( + env_key="MOONSHOT_API_KEY", + default_base_url="https://api.moonshot.ai/v1", + supports_platform_proxy=False, + supports_chat_reasoning_content=True, + ), + "sambanova": _ProviderConfig( + env_key="SAMBANOVA_API_KEY", + default_base_url="https://api.sambanova.ai/v1", + flatten_text_content=True, + ), +} diff --git a/python/timbal/core/llm/responses.py b/python/timbal/core/llm/responses.py new file mode 100644 index 00000000..36709f9a --- /dev/null +++ b/python/timbal/core/llm/responses.py @@ -0,0 +1,77 @@ +"""OpenAI Responses API request adapter (openai, xai).""" + +from collections.abc import AsyncIterator, Callable +from typing import TYPE_CHECKING, Any + +from ...types.message import Message +from ...utils import transform_schema + +if TYPE_CHECKING: + from pydantic import BaseModel + + from ..runnable import Runnable + + +def prepare_responses_request( + *, + client: Any, + model_name: str, + request_headers: dict[str, str], + system_prompt: str | None, + messages: list[Message], + tools: "list[Runnable] | None", + max_tokens: int | None, + temperature: float | None, + output_model: "type[BaseModel] | None", + provider_params: dict[str, Any], +) -> tuple[Callable[[], AsyncIterator[Any]], str]: + """Build the Responses API kwargs and return (create_stream, context_label).""" + responses_kwargs = { + "model": model_name, + "stream": True, + "store": False, + "include": ["web_search_call.action.sources"], + } + + if system_prompt: + responses_kwargs["instructions"] = system_prompt + + responses_kwargs["input"] = sum([message.to_openai_responses_input() for message in messages], []) + + if tools: + responses_tools = [tool.openai_responses_schema for tool in tools] + if responses_tools: + responses_kwargs["tools"] = responses_tools + responses_kwargs["parallel_tool_calls"] = True + + if max_tokens: + responses_kwargs["max_output_tokens"] = max_tokens + + if temperature is not None: + responses_kwargs["temperature"] = temperature + + if output_model is not None: + responses_kwargs["text"] = { + "format": { + "type": "json_schema", + "name": output_model.__name__, + "schema": transform_schema(output_model), + "strict": True, + } + } + + # provider_params may carry provider-native (server-side) tool defs, e.g. + # {"type": "web_search"}. Merge them with the client tools instead of + # letting dict.update clobber the generated list. + provider_params = dict(provider_params) + extra_tools = provider_params.pop("tools", None) + responses_kwargs.update(provider_params) + if extra_tools: + responses_kwargs["tools"] = [*responses_kwargs.get("tools", []), *extra_tools] + + async def _create_stream(): + res = await client.responses.create(extra_headers=request_headers, **responses_kwargs) + async for chunk in res: + yield chunk + + return _create_stream, "OpenAI Responses" diff --git a/python/timbal/core/llm/retry.py b/python/timbal/core/llm/retry.py new file mode 100644 index 00000000..a0a5a69f --- /dev/null +++ b/python/timbal/core/llm/retry.py @@ -0,0 +1,145 @@ +"""Retry loop for transient provider failures (rate limits, timeouts, 5xx).""" + +import asyncio +import random + +import structlog + +from ..provider_errors import provider_error_classes + +logger = structlog.get_logger("timbal.core.llm") + +MAX_RETRY_DELAY = 30.0 + + +async def _retry_on_error(async_gen_func, max_retries: int, retry_delay: float, context: str): + """Helper to retry an async generator function on transient failures. + + Retryable errors (using SDK exception types): + - Empty streams (StopAsyncIteration) + - Rate limiting (RateLimitError from OpenAI/Anthropic SDKs) + - Timeouts (APITimeoutError from OpenAI/Anthropic SDKs) + - Connection errors (APIConnectionError from OpenAI/Anthropic SDKs) + - Server errors (APIStatusError with 500, 502, 503, 504 status codes) + - Overloaded/capacity errors (APIError with "overload" or "capacity" in message) + + Non-retryable errors (fail immediately): + - Authentication errors (401, 403) + - Invalid requests (400, 404) + - Other 4xx client errors + + Args: + async_gen_func: Async callable that returns an async generator + max_retries: Maximum number of retry attempts + retry_delay: Base delay for exponential backoff + context: Description for logging (e.g., "Anthropic API") + + Yields: + Items from the async generator + + Raises: + Exception: Original exception if not retryable or max retries exceeded + """ + last_error = None + + for attempt in range(max_retries + 1): + try: + async_gen = async_gen_func() + # Try to get the first item to detect empty streams + first_item = await async_gen.__anext__() + # Success - yield first item and then all remaining items + yield first_item + async for item in async_gen: + yield item + return # Successfully completed + + except StopAsyncIteration as e: + # Empty stream detected + last_error = e + error_type = "empty_stream" + error_msg = "Empty stream" + + except Exception as e: + # Check if it's a retryable error + last_error = e + error_type = type(e).__name__ + error_msg = str(e) + + # Determine if error is retryable based on SDK exception types + # (classes resolved lazily — we're on an LLM error path, the SDK + # that raised is already imported). + is_retryable = False + err_cls = provider_error_classes() + + if isinstance(e, err_cls["rate_limit"]): + is_retryable = True + error_type = "rate_limit" + + elif isinstance(e, err_cls["timeout"]): + is_retryable = True + error_type = "timeout" + + elif isinstance(e, err_cls["connection"]): + is_retryable = True + error_type = "connection_error" + + elif isinstance(e, err_cls["status"]): + # Check status code for retryable HTTP errors + status_code = getattr(e, "status_code", None) + if status_code in [429, 500, 502, 503, 504]: + is_retryable = True + if status_code == 429: + error_type = "rate_limit" + elif status_code == 503: + error_type = "service_unavailable" + else: + error_type = f"server_error_{status_code}" + # Don't retry on 4xx errors (client errors like 400, 401, 403, 404) + + # If not retryable, re-raise immediately + if not is_retryable: + logger.error( + "Non-retryable error from LLM provider", context=context, error_type=error_type, error=error_msg + ) + raise + + # Retry logic for retryable errors + if attempt < max_retries: + cap = min(retry_delay * (2**attempt), MAX_RETRY_DELAY) + delay = random.uniform(0, cap) + retry_after = _retry_after_seconds(last_error) + if retry_after is not None: + delay = max(delay, retry_after) + logger.warning( + "Retryable error from LLM provider, retrying...", + context=context, + error_type=error_type, + error=error_msg, + attempt=attempt + 1, + max_retries=max_retries, + delay=delay, + ) + await asyncio.sleep(delay) + else: + # Max retries exceeded + logger.error( + "Max retries exceeded for LLM provider", context=context, error_type=error_type, max_retries=max_retries + ) + # Re-raise the last error + raise last_error + + +def _retry_after_seconds(exc: BaseException | None) -> float | None: + if exc is None: + return None + response = getattr(exc, "response", None) + headers = getattr(response, "headers", None) + if not headers: + return None + retry_after = headers.get("Retry-After") + if retry_after is None: + return None + try: + return float(retry_after) + except (TypeError, ValueError): + return None diff --git a/python/timbal/core/llm/router.py b/python/timbal/core/llm/router.py new file mode 100644 index 00000000..98528c79 --- /dev/null +++ b/python/timbal/core/llm/router.py @@ -0,0 +1,145 @@ +"""Model-string dispatch: fallback/TestModel short-circuits, client resolution, +file loading, and per-API request dispatch through the shared retry loop.""" + +import asyncio +from typing import Any + +from pydantic import BaseModel, SecretStr + +from ...state import get_call_id, get_or_create_run_context, set_billing_id +from ...types.message import Message +from ..runnable import Runnable +from .chat_completions import prepare_chat_completions_request +from .clients import _get_file_client, _resolve_client +from .messages import prepare_messages_request +from .registry import _PROVIDERS, TIMBAL_OPENAI_API +from .responses import prepare_responses_request +from .retry import _retry_on_error + + +async def _llm_router( + model: Any, # Model | str | TestModel — typed as Any so Pydantic doesn't reject TestModel instances + system_prompt: str | None = None, + messages: list[Message] | None = None, + tools: list[Runnable] | None = None, + max_tokens: int | None = None, + temperature: float | None = None, + output_model: type[BaseModel] | None = None, + base_url: str | SecretStr | None = None, + api_key: str | SecretStr | None = None, + max_retries: int = 0, + retry_delay: float = 1.0, + provider_params: dict[str, Any] | None = None, +) -> Message: # type: ignore[misc] # Declared as Message for framework schema generation; runtime is an async generator of provider-specific chunks. + """ + Internal LLM router function. + + WARNING: This function is for internal use only and may change frequently + as LLM providers update their APIs. Use the stable Agent/Workflow APIs instead. + """ + messages = messages or [] + provider_params = provider_params or {} + + if getattr(model, "__timbal_fallback_model__", False): + async for chunk in model.route( + _llm_router, + system_prompt=system_prompt, + messages=messages, + tools=tools, + max_tokens=max_tokens, + temperature=temperature, + output_model=output_model, + base_url=base_url, + api_key=api_key, + max_retries=max_retries, + retry_delay=retry_delay, + provider_params=provider_params, + ): + yield chunk # type: ignore[return-type] + return + + # Convert SecretStr to str if needed + if isinstance(base_url, SecretStr): + base_url = base_url.get_secret_value() + if isinstance(api_key, SecretStr): + api_key = api_key.get_secret_value() + + # TestModel short-circuit — delegates to model.stream() with no network call. + if hasattr(model, "stream"): + async for chunk in model.stream(messages=messages): + yield chunk # type: ignore[return-type] + return + + if "/" not in model: + raise ValueError("Model must be in format 'provider/model_name'") + + set_billing_id(model) + provider, model_name = model.split("/", 1) + + config = _PROVIDERS.get(provider) + if config is None: + raise ValueError(f"Unsupported provider: {provider}") + + # Anthropic requires max_tokens + if provider == "anthropic" and not max_tokens: + raise ValueError("'max_tokens' is required for claude models.") + + run_context = get_or_create_run_context() + call_id = get_call_id() + # Per-request headers: change every call, so passed via extra_headers on each .create(). + request_headers: dict[str, str] = { + "x-timbal-run-id": run_context.id, + } + if call_id: + request_headers["x-timbal-call-id"] = call_id + if run_context.platform_config and run_context.platform_config.subject: + if run_context.platform_config.subject.app_id: + request_headers["x-timbal-app-id"] = run_context.platform_config.subject.app_id + + client, base_url = _resolve_client(provider, config, api_key, base_url, run_context) + + # Eagerly load all unloaded file content (async, concurrent) before + # serialization. This scan lives here — not inside to_*_input() — because: + # 1. gather() needs the full list upfront for concurrent downloads. + # 2. to_*_input() stays sync and pure (format conversion, no I/O). + # The double iteration (scan here + serialize in to_*_input) is intentional: + # content arrays are small (1-5 items) and the cost is negligible vs the + # network calls that follow. + from ...types.content import FileContent + + _unloaded_files = [ + c.file + for m in messages + for c in m.content + if isinstance(c, FileContent) and object.__getattribute__(c.file, "__fileobj__") is None + ] + if _unloaded_files: + await asyncio.gather(*(f.load(client=_get_file_client()) for f in _unloaded_files)) + + # Per-API request builders return the stream factory; the single retry + # loop below is shared, so the per-chunk generator nesting is identical + # to an inline implementation. + request_kwargs: dict[str, Any] = { + "client": client, + "model_name": model_name, + "request_headers": request_headers, + "system_prompt": system_prompt, + "messages": messages, + "tools": tools, + "max_tokens": max_tokens, + "temperature": temperature, + "output_model": output_model, + "provider_params": provider_params, + } + if provider == "anthropic": + create_stream, context = prepare_messages_request(**request_kwargs) + elif provider in ("openai", "xai") and TIMBAL_OPENAI_API == "responses": + create_stream, context = prepare_responses_request(**request_kwargs) + else: + # OpenAI Chat Completions compatible providers + create_stream, context = prepare_chat_completions_request( + provider=provider, config=config, **request_kwargs, + ) + + async for res_chunk in _retry_on_error(create_stream, max_retries, retry_delay, context): + yield res_chunk # type: ignore[return-type] diff --git a/python/timbal/core/llm_router.py b/python/timbal/core/llm_router.py deleted file mode 100644 index 40157cc5..00000000 --- a/python/timbal/core/llm_router.py +++ /dev/null @@ -1,641 +0,0 @@ -""" -INTERNAL USE ONLY - -This module is intended for internal use and will be subject to frequent changes -as LLM providers constantly update their APIs and add new features. The external -APIs (Runnables, Agents, Workflows) will remain stable, but this module will -evolve to match provider changes. - -Do not rely on this module's interface in external code. -""" - -import asyncio -import os -import random -from dataclasses import dataclass -from types import SimpleNamespace -from typing import Any, Literal - -import structlog -from anthropic import APIConnectionError as AnthropicAPIConnectionError -from anthropic import APIStatusError as AnthropicAPIStatusError -from anthropic import APITimeoutError as AnthropicAPITimeoutError -from anthropic import AsyncAnthropic -from anthropic import RateLimitError as AnthropicRateLimitError # APIError as AnthropicAPIError, -from openai import APIConnectionError as OpenAIAPIConnectionError -from openai import APIStatusError as OpenAIAPIStatusError -from openai import APITimeoutError as OpenAIAPITimeoutError -from openai import AsyncOpenAI -from openai import RateLimitError as OpenAIRateLimitError # APIError as OpenAIAPIError, -from pydantic import BaseModel, SecretStr - -from ..errors import APIKeyNotFoundError -from ..state import get_call_id, get_or_create_run_context, set_billing_id -from ..types.message import Message -from ..utils import transform_schema -from .runnable import Runnable - -logger = structlog.get_logger("timbal.core.llm_router") -MAX_RETRY_DELAY = 30.0 - -# Module-level client cache keyed by (client_class, api_key, base_url, provider). -# Reusing clients preserves the underlying httpx connection pool, avoiding a -# fresh TCP+TLS handshake on every LLM call (~200-300ms saved per request). -# Per-request tracing headers (run_id, call_id) are passed via extra_headers -# on each individual .create() call instead. -# Concurrency-safe: all coroutines run on one thread in asyncio; the GIL -# ensures check-and-assign is atomic (no await between the if and the write). -_CLIENT_CACHE: dict[tuple, AsyncOpenAI | AsyncAnthropic] = {} - -# Shared httpx client for async file loading. Reused across LLM calls to -# preserve connection pools to file origins (CDNs, S3, etc.). Lazy-initialized -# on first use — never created if the conversation has no files. -# Loop-aware: httpx.AsyncClient is bound to the event loop at creation time. -# When the loop changes (e.g. between pytest-asyncio tests with per-function -# loop scope), the old client becomes unusable ("Event loop is closed"). -# We detect this and transparently recreate the client. -_FILE_CLIENT: Any = None -_FILE_CLIENT_LOOP: Any = None - - -def _get_file_client() -> Any: - global _FILE_CLIENT, _FILE_CLIENT_LOOP - loop = asyncio.get_running_loop() - if _FILE_CLIENT is not None and (_FILE_CLIENT.is_closed or _FILE_CLIENT_LOOP is not loop): - _FILE_CLIENT = None - if _FILE_CLIENT is None: - import httpx - - _FILE_CLIENT = httpx.AsyncClient(timeout=httpx.Timeout(30.0)) - _FILE_CLIENT_LOOP = loop - return _FILE_CLIENT - - -def _get_client(cls: type, api_key: str, base_url: str | None, provider: str) -> AsyncOpenAI | AsyncAnthropic: - cache_key = (cls, api_key, base_url, provider) - if cache_key not in _CLIENT_CACHE: - kwargs: dict[str, Any] = {"api_key": api_key, "default_headers": {"x-provider": provider}} - if base_url: - kwargs["base_url"] = base_url - _CLIENT_CACHE[cache_key] = cls(**kwargs) - return _CLIENT_CACHE[cache_key] - - -TIMBAL_OPENAI_API = os.getenv("TIMBAL_OPENAI_API", "responses") -if TIMBAL_OPENAI_API != "responses": - logger.warning( - "Using legacy Chat Completions API. OpenAI is transitioning to the new Responses API, " - "which should be preferred for all new development. Set TIMBAL_OPENAI_API=responses to switch." - ) - - -# --------------------------------------------------------------------------- -# Provider configuration -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True, slots=True) -class _ProviderConfig: - """Static configuration for a single LLM provider.""" - - env_key: str - """Environment variable name for the API key (e.g. ``OPENAI_API_KEY``).""" - - default_base_url: str | None = None - """Default API base URL. ``None`` uses the SDK default.""" - - proxy_name: str = "openai-completions" - """Platform proxy path segment (e.g. ``openai-responses``, ``anthropic``).""" - - proxy_suffix: str = "/v1" - """Appended to the proxy URL. Anthropic uses ``""``.""" - - client_type: Literal["openai", "anthropic"] = "openai" - """Which SDK client to create.""" - - flatten_text_content: bool = False - """Flatten text-only content arrays to plain strings for providers with incomplete Chat Completions support.""" - - supports_stream_options: bool = True - """Whether the provider supports ``stream_options`` in Chat Completions.""" - - supports_platform_proxy: bool = True - """If False, never fall back to the Timbal platform proxy — require the provider API key.""" - - supports_chat_reasoning_content: bool = False - """If True, serialize ThinkingContent as top-level ``reasoning_content`` (Moonshot/Fireworks-style). - - Otherwise thinking is omitted from outbound chat-completions messages (Vercel/LiteLLM default). - """ - - -_PROVIDERS: dict[str, _ProviderConfig] = { - "openai": _ProviderConfig( - env_key="OPENAI_API_KEY", - proxy_name="openai-responses" if TIMBAL_OPENAI_API == "responses" else "openai-completions", - ), - "anthropic": _ProviderConfig( - env_key="ANTHROPIC_API_KEY", - proxy_name="anthropic", - proxy_suffix="", - client_type="anthropic", - ), - "google": _ProviderConfig( - env_key="GEMINI_API_KEY", - default_base_url="https://generativelanguage.googleapis.com/v1beta/openai/", - ), - "togetherai": _ProviderConfig( - env_key="TOGETHER_API_KEY", - default_base_url="https://api.together.xyz/v1/", - supports_chat_reasoning_content=True, - ), - "xai": _ProviderConfig( - env_key="XAI_API_KEY", - default_base_url="https://api.x.ai/v1", - proxy_name="openai-responses", - ), - "groq": _ProviderConfig( - env_key="GROQ_API_KEY", - default_base_url="https://api.groq.com/openai/v1", - ), - "fireworks": _ProviderConfig( - env_key="FIREWORKS_API_KEY", - default_base_url="https://api.fireworks.ai/inference/v1", - supports_chat_reasoning_content=True, - ), - "byteplus": _ProviderConfig( - env_key="BYTEPLUS_API_KEY", - default_base_url="https://ark.ap-southeast.bytepluses.com/api/v3", - supports_chat_reasoning_content=True, - ), - "xiaomi": _ProviderConfig( - env_key="XIAOMI_API_KEY", - default_base_url="https://api.xiaomimimo.com/v1", - flatten_text_content=True, - supports_stream_options=False, - ), - "cerebras": _ProviderConfig( - env_key="CEREBRAS_API_KEY", - default_base_url="https://api.cerebras.ai/v1", - ), - "moonshot": _ProviderConfig( - env_key="MOONSHOT_API_KEY", - default_base_url="https://api.moonshot.ai/v1", - supports_platform_proxy=False, - supports_chat_reasoning_content=True, - ), - "sambanova": _ProviderConfig( - env_key="SAMBANOVA_API_KEY", - default_base_url="https://api.sambanova.ai/v1", - flatten_text_content=True, - ), -} - - -def _resolve_client( - provider: str, - config: _ProviderConfig, - api_key: str | None, - base_url: str | None, - run_context: Any, -) -> tuple[AsyncOpenAI | AsyncAnthropic, str | None]: - """Resolve API key, base URL, and return the appropriate SDK client. - - Returns: - (client, resolved_base_url) — base_url may have been updated for platform proxies. - """ - if not api_key: - api_key = os.getenv(config.env_key) - if not api_key: - if ( - config.supports_platform_proxy - and run_context.platform_config is not None - and run_context.platform_config.subject is not None - ): - api_key = run_context.platform_config.auth.header_value - base_url = ( - f"https://{run_context.platform_config.host}" - f"/orgs/{run_context.platform_config.subject.org_id}" - f"/proxies/{config.proxy_name}{config.proxy_suffix}" - ) - if not api_key: - raise APIKeyNotFoundError(f"{config.env_key} not found.") - - if config.client_type == "anthropic": - return _get_client(AsyncAnthropic, api_key, base_url, "anthropic"), base_url - return _get_client(AsyncOpenAI, api_key, base_url or config.default_base_url, provider), base_url - - -async def warmup_llm_connection(model: str) -> None: - """Pre-establish the provider's HTTPS connection pool for ``model``. - - The first LLM call of a process pays TCP+TLS(+HTTP/2) setup before any - token arrives — measured ~1.3s extra TTFT cold vs warm against - api.openai.com. This issues one lightweight authenticated GET - (``/models``) through the same cached SDK client the real calls will use, - so the pool is hot by the time the first request fires. - - Best-effort and side-effect free: any failure (no API key, unsupported - endpoint on OpenAI-compatible providers, timeout) is logged at DEBUG and - ignored. Callers fire-and-forget (e.g. voice sessions at startup). - """ - try: - provider, _ = model.split("/", 1) - except ValueError: - return - config = _PROVIDERS.get(provider) - if config is None: - return - try: - # _resolve_client only reads run_context.platform_config; don't create - # a real RunContext here (warmup must not touch tracing state). - ctx = SimpleNamespace(platform_config=None) - client, _ = _resolve_client(provider, config, None, None, ctx) - await asyncio.wait_for(client.models.list(), timeout=5.0) - logger.debug("llm_connection_warmed", model=model) - except Exception as e: - logger.debug("llm_warmup_skipped", model=model, error=str(e)) - - -# --------------------------------------------------------------------------- -# Retry helper -# --------------------------------------------------------------------------- - - -async def _retry_on_error(async_gen_func, max_retries: int, retry_delay: float, context: str): - """Helper to retry an async generator function on transient failures. - - Retryable errors (using SDK exception types): - - Empty streams (StopAsyncIteration) - - Rate limiting (RateLimitError from OpenAI/Anthropic SDKs) - - Timeouts (APITimeoutError from OpenAI/Anthropic SDKs) - - Connection errors (APIConnectionError from OpenAI/Anthropic SDKs) - - Server errors (APIStatusError with 500, 502, 503, 504 status codes) - - Overloaded/capacity errors (APIError with "overload" or "capacity" in message) - - Non-retryable errors (fail immediately): - - Authentication errors (401, 403) - - Invalid requests (400, 404) - - Other 4xx client errors - - Args: - async_gen_func: Async callable that returns an async generator - max_retries: Maximum number of retry attempts - retry_delay: Base delay for exponential backoff - context: Description for logging (e.g., "Anthropic API") - - Yields: - Items from the async generator - - Raises: - Exception: Original exception if not retryable or max retries exceeded - """ - last_error = None - - for attempt in range(max_retries + 1): - try: - async_gen = async_gen_func() - # Try to get the first item to detect empty streams - first_item = await async_gen.__anext__() - # Success - yield first item and then all remaining items - yield first_item - async for item in async_gen: - yield item - return # Successfully completed - - except StopAsyncIteration as e: - # Empty stream detected - last_error = e - error_type = "empty_stream" - error_msg = "Empty stream" - - except Exception as e: - # Check if it's a retryable error - last_error = e - error_type = type(e).__name__ - error_msg = str(e) - - # Determine if error is retryable based on SDK exception types - is_retryable = False - - # OpenAI SDK exceptions - if isinstance(e, (OpenAIRateLimitError, AnthropicRateLimitError)): - is_retryable = True - error_type = "rate_limit" - - elif isinstance(e, (OpenAIAPITimeoutError, AnthropicAPITimeoutError)): - is_retryable = True - error_type = "timeout" - - elif isinstance(e, (OpenAIAPIConnectionError, AnthropicAPIConnectionError)): - is_retryable = True - error_type = "connection_error" - - elif isinstance(e, (OpenAIAPIStatusError, AnthropicAPIStatusError)): - # Check status code for retryable HTTP errors - status_code = getattr(e, "status_code", None) - if status_code in [429, 500, 502, 503, 504]: - is_retryable = True - if status_code == 429: - error_type = "rate_limit" - elif status_code == 503: - error_type = "service_unavailable" - else: - error_type = f"server_error_{status_code}" - # Don't retry on 4xx errors (client errors like 400, 401, 403, 404) - - # If not retryable, re-raise immediately - if not is_retryable: - logger.error( - "Non-retryable error from LLM provider", context=context, error_type=error_type, error=error_msg - ) - raise - - # Retry logic for retryable errors - if attempt < max_retries: - cap = min(retry_delay * (2**attempt), MAX_RETRY_DELAY) - delay = random.uniform(0, cap) - retry_after = _retry_after_seconds(last_error) - if retry_after is not None: - delay = max(delay, retry_after) - logger.warning( - "Retryable error from LLM provider, retrying...", - context=context, - error_type=error_type, - error=error_msg, - attempt=attempt + 1, - max_retries=max_retries, - delay=delay, - ) - await asyncio.sleep(delay) - else: - # Max retries exceeded - logger.error( - "Max retries exceeded for LLM provider", context=context, error_type=error_type, max_retries=max_retries - ) - # Re-raise the last error - raise last_error - - -def _retry_after_seconds(exc: BaseException | None) -> float | None: - if exc is None: - return None - response = getattr(exc, "response", None) - headers = getattr(response, "headers", None) - if not headers: - return None - retry_after = headers.get("Retry-After") - if retry_after is None: - return None - try: - return float(retry_after) - except (TypeError, ValueError): - return None - - -# --------------------------------------------------------------------------- -# Main router -# --------------------------------------------------------------------------- - - -async def _llm_router( - model: Any, # Model | str | TestModel — typed as Any so Pydantic doesn't reject TestModel instances - system_prompt: str | None = None, - messages: list[Message] | None = None, - tools: list[Runnable] | None = None, - max_tokens: int | None = None, - temperature: float | None = None, - output_model: type[BaseModel] | None = None, - base_url: str | SecretStr | None = None, - api_key: str | SecretStr | None = None, - max_retries: int = 0, - retry_delay: float = 1.0, - provider_params: dict[str, Any] | None = None, -) -> Message: # type: ignore[misc] # Declared as Message for framework schema generation; runtime is an async generator of provider-specific chunks. - """ - Internal LLM router function. - - WARNING: This function is for internal use only and may change frequently - as LLM providers update their APIs. Use the stable Agent/Workflow APIs instead. - """ - messages = messages or [] - provider_params = provider_params or {} - - if getattr(model, "__timbal_fallback_model__", False): - async for chunk in model.route( - _llm_router, - system_prompt=system_prompt, - messages=messages, - tools=tools, - max_tokens=max_tokens, - temperature=temperature, - output_model=output_model, - base_url=base_url, - api_key=api_key, - max_retries=max_retries, - retry_delay=retry_delay, - provider_params=provider_params, - ): - yield chunk # type: ignore[return-type] - return - - # Convert SecretStr to str if needed - if isinstance(base_url, SecretStr): - base_url = base_url.get_secret_value() - if isinstance(api_key, SecretStr): - api_key = api_key.get_secret_value() - - # TestModel short-circuit — delegates to model.stream() with no network call. - if hasattr(model, "stream"): - async for chunk in model.stream(messages=messages): - yield chunk # type: ignore[return-type] - return - - if "/" not in model: - raise ValueError("Model must be in format 'provider/model_name'") - - set_billing_id(model) - provider, model_name = model.split("/", 1) - - config = _PROVIDERS.get(provider) - if config is None: - raise ValueError(f"Unsupported provider: {provider}") - - # Anthropic requires max_tokens - if provider == "anthropic" and not max_tokens: - raise ValueError("'max_tokens' is required for claude models.") - - run_context = get_or_create_run_context() - call_id = get_call_id() - # Per-request headers: change every call, so passed via extra_headers on each .create(). - request_headers: dict[str, str] = { - "x-timbal-run-id": run_context.id, - } - if call_id: - request_headers["x-timbal-call-id"] = call_id - if run_context.platform_config and run_context.platform_config.subject: - if run_context.platform_config.subject.app_id: - request_headers["x-timbal-app-id"] = run_context.platform_config.subject.app_id - - client, base_url = _resolve_client(provider, config, api_key, base_url, run_context) - - # Eagerly load all unloaded file content (async, concurrent) before - # serialization. This scan lives here — not inside to_*_input() — because: - # 1. gather() needs the full list upfront for concurrent downloads. - # 2. to_*_input() stays sync and pure (format conversion, no I/O). - # The double iteration (scan here + serialize in to_*_input) is intentional: - # content arrays are small (1-5 items) and the cost is negligible vs the - # network calls that follow. - from ..types.content import FileContent - - _unloaded_files = [ - c.file - for m in messages - for c in m.content - if isinstance(c, FileContent) and object.__getattribute__(c.file, "__fileobj__") is None - ] - if _unloaded_files: - await asyncio.gather(*(f.load(client=_get_file_client()) for f in _unloaded_files)) - - if provider == "anthropic": - anthropic_kwargs = { - "model": model_name, - "messages": [message.to_anthropic_input() for message in messages], - "max_tokens": max_tokens, - "stream": True, - } - - if system_prompt: - anthropic_kwargs["system"] = system_prompt - - if tools: - anthropic_tools = [tool.anthropic_schema for tool in tools] - if anthropic_tools: - anthropic_kwargs["tools"] = anthropic_tools - - if temperature is not None: - anthropic_kwargs["temperature"] = temperature - - anthropic_kwargs.update(provider_params) - - async def _create_stream(): - if output_model is not None: - anthropic_kwargs["output_config"] = { - "format": { - "type": "json_schema", - "schema": transform_schema(output_model), - } - } - res = await client.messages.create(extra_headers=request_headers, **anthropic_kwargs) # type: ignore[attr-defined] - async for chunk in res: - yield chunk - - async for res_chunk in _retry_on_error(_create_stream, max_retries, retry_delay, "Anthropic"): - yield res_chunk # type: ignore[return-type] - - elif provider in ("openai", "xai") and TIMBAL_OPENAI_API == "responses": - responses_kwargs = { - "model": model_name, - "stream": True, - "store": False, - "include": ["web_search_call.action.sources"], - } - - if system_prompt: - responses_kwargs["instructions"] = system_prompt - - responses_kwargs["input"] = sum([message.to_openai_responses_input() for message in messages], []) - - if tools: - responses_tools = [tool.openai_responses_schema for tool in tools] - if responses_tools: - responses_kwargs["tools"] = responses_tools - responses_kwargs["parallel_tool_calls"] = True - - if max_tokens: - responses_kwargs["max_output_tokens"] = max_tokens - - if temperature is not None: - responses_kwargs["temperature"] = temperature - - if output_model is not None: - responses_kwargs["text"] = { - "format": { - "type": "json_schema", - "name": output_model.__name__, - "schema": transform_schema(output_model), - "strict": True, - } - } - - responses_kwargs.update(provider_params) - - async def _create_stream(): - res = await client.responses.create(extra_headers=request_headers, **responses_kwargs) # type: ignore[attr-defined] - async for chunk in res: - yield chunk - - async for res_chunk in _retry_on_error(_create_stream, max_retries, retry_delay, "OpenAI Responses"): - yield res_chunk # type: ignore[return-type] - - # OpenAI Chat Completions compatible providers - else: - chat_completions_messages = [] - if system_prompt: - chat_completions_messages.append({"role": "system", "content": system_prompt}) - reasoning_as = "reasoning_content" if config.supports_chat_reasoning_content else "omit" - for message in messages: - chat_completions_message = message.to_openai_chat_completions_input(reasoning_as=reasoning_as) - chat_completions_messages.append(chat_completions_message) - - # Some providers have incomplete OpenAI chat completions support. - # Flatten text-only content arrays to plain strings for compatibility. - if config.flatten_text_content: - for msg in chat_completions_messages: - content = msg.get("content") - if isinstance(content, list) and all( - isinstance(item, dict) and item.get("type") == "text" for item in content - ): - msg["content"] = "\n".join(item["text"] for item in content) - - chat_completions_kwargs: dict[str, Any] = { - "model": model_name, - "messages": chat_completions_messages, - "stream": True, - } - - if config.supports_stream_options: - chat_completions_kwargs["stream_options"] = {"include_usage": True} - - if tools: - chat_completions_tools = [tool.openai_chat_completions_schema for tool in tools] - if chat_completions_tools: - chat_completions_kwargs["tools"] = chat_completions_tools - - if max_tokens: - chat_completions_kwargs["max_completion_tokens"] = max_tokens - - if temperature is not None: - chat_completions_kwargs["temperature"] = temperature - - if output_model is not None: - chat_completions_kwargs["response_format"] = { - "type": "json_schema", - "json_schema": { - "name": output_model.__name__, - "schema": transform_schema(output_model), - "strict": True, - }, - } - - chat_completions_kwargs.update(provider_params) - - async def _create_stream(): - res = await client.chat.completions.create(extra_headers=request_headers, **chat_completions_kwargs) # type: ignore[attr-defined] - async for chunk in res: - yield chunk - - async for res_chunk in _retry_on_error( - _create_stream, max_retries, retry_delay, f"{provider} Chat Completions" - ): - yield res_chunk # type: ignore[return-type] diff --git a/python/timbal/core/memory_compaction.py b/python/timbal/core/memory_compaction.py index 9bda173f..c8dc401d 100644 --- a/python/timbal/core/memory_compaction.py +++ b/python/timbal/core/memory_compaction.py @@ -204,7 +204,7 @@ async def _call_summarizer(model: Any, prompt: str, max_summary_tokens: int) -> import time from ..collectors import get_collector_registry - from .llm_router import _llm_router + from .llm import _llm_router prompt_message = Message.validate({"role": "user", "content": prompt}) chunks = _llm_router( diff --git a/python/timbal/core/provider_errors.py b/python/timbal/core/provider_errors.py new file mode 100644 index 00000000..8a3f6ce3 --- /dev/null +++ b/python/timbal/core/provider_errors.py @@ -0,0 +1,50 @@ +"""Lazy access to provider-SDK exception classes. + +Importing ``openai`` + ``anthropic`` costs ~460 ms — a third of the entire +``from timbal import Agent`` import — and these classes are only needed to +*classify* errors at LLM-call time. + +Classification never imports an SDK: an exception can only be an instance of +a class from an SDK that is already in ``sys.modules`` (it raised it), so the +tuples are built from whichever provider SDKs the process has actually used. +An Anthropic-only process therefore never loads ``openai`` at all, and vice +versa. Rebuilt per call (a few attribute lookups, error paths only) so a +fallback chain that touches a second provider later is classified correctly. +""" + +import sys + + +def provider_error_classes() -> dict[str, tuple[type, ...]]: + """Exception-class tuples for retry/fallback classification. + + Keys: ``rate_limit``, ``timeout``, ``connection``, ``status``. Each value + contains the classes from the provider SDKs currently imported — possibly + empty tuples (``isinstance(e, ())`` is False), e.g. for TestModel-only + processes where no provider SDK ever loads. + """ + rate_limit: list[type] = [] + timeout: list[type] = [] + connection: list[type] = [] + status: list[type] = [] + + openai = sys.modules.get("openai") + if openai is not None: + rate_limit.append(openai.RateLimitError) + timeout.append(openai.APITimeoutError) + connection.append(openai.APIConnectionError) + status.append(openai.APIStatusError) + + anthropic = sys.modules.get("anthropic") + if anthropic is not None: + rate_limit.append(anthropic.RateLimitError) + timeout.append(anthropic.APITimeoutError) + connection.append(anthropic.APIConnectionError) + status.append(anthropic.APIStatusError) + + return { + "rate_limit": tuple(rate_limit), + "timeout": tuple(timeout), + "connection": tuple(connection), + "status": tuple(status), + } diff --git a/python/timbal/core/runnable.py b/python/timbal/core/runnable.py index 209fdce6..68759950 100644 --- a/python/timbal/core/runnable.py +++ b/python/timbal/core/runnable.py @@ -4,6 +4,7 @@ import hashlib import inspect import json +import logging import os import secrets import time @@ -17,9 +18,7 @@ BaseModel, ConfigDict, Field, - PrivateAttr, TypeAdapter, - ValidationInfo, computed_field, field_validator, model_serializer, @@ -60,7 +59,7 @@ from ..types.events.delta import Custom, DeltaEvent, DeltaItem from ..types.message import Message from ..types.run_status import RunStatus -from ..utils import dump, sync_to_async_gen +from ..utils import dump, invalidate_message_dump_caches, sync_to_async_gen def _get_logger(): @@ -69,6 +68,21 @@ def _get_logger(): return structlog.get_logger("timbal.core.runnable") +_stdlib_events_logger = logging.getLogger("timbal.core.runnable") + + +def _events_logging_enabled() -> bool: + """Cheap gate for per-event INFO logging. + + ``event.model_dump()`` is expensive and is evaluated eagerly as a call + argument, so callers must check this BEFORE building the log kwargs. + Timbal's structlog setup is stdlib-backed (see logs.setup_logging), so the + stdlib effective level (which also respects ``logging.disable``) is + authoritative. + """ + return _stdlib_events_logger.isEnabledFor(logging.INFO) + + def _collector_output_on_interrupt(collector: Any) -> Any: """Best-effort partial output when the handler async generator stops abruptly. @@ -100,34 +114,42 @@ def _collector_output_on_interrupt(collector: Any) -> Any: return raw -def _emit_default_tool_usage(runnable: Any) -> None: - """On successful Tool completion, record ``{tool.name}:requests`` for billing defaults.""" - from .tool import Tool +_Tool = None +_record_tool_requests = None - if not isinstance(runnable, Tool): - return - if not getattr(runnable, "record_default_request_usage", True): - return - from ..state import _record_tool_requests - _record_tool_requests(runnable.name) +def _emit_default_tool_usage(runnable: Any) -> None: + """On successful Tool completion, record ``{tool.name}:requests`` for billing defaults. + Runs on every successful call — the imports are cached in module globals + after the first invocation. + """ + global _Tool, _record_tool_requests + if _Tool is None: + from ..state import _record_tool_requests as _record_fn + from .tool import Tool -def _timbal_collector_wrap(fn): - """Lazy wrapper for @TimbalCollector.wrap — avoids importing the collector at module load.""" - from functools import wraps + _Tool = Tool + _record_tool_requests = _record_fn - @wraps(fn) - def wrapper(self, **kwargs): - from ..collectors.impl.timbal import TimbalCollector + if not isinstance(runnable, _Tool): + return + if not runnable.record_default_request_usage: + return + _record_tool_requests(runnable.name) - return TimbalCollector(async_gen=fn(self, **kwargs)) - return wrapper +_TimbalCollector = None +"""Lazily imported TimbalCollector class — avoids importing collectors at module load.""" ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789" +_BLOCKING_HANDLER_WARN_MS = float(os.getenv("TIMBAL_BLOCKING_WARN_MS", "100")) +"""Sync handlers running inline for longer than this (ms) get a one-time +warning suggesting offload_blocking=True / an async handler. Set to 0 to warn +on any sync handler; raise it (or set very high) to silence.""" + ApprovalPolicy = bool | Callable[..., bool | ApprovalPolicyDecision | dict[str, Any]] ApprovalPrompt = str | Callable[..., str | None] | None @@ -211,8 +233,6 @@ class Runnable(ABC, BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow") - _is_timbal_runnable: bool = True # Marker for fast isinstance check in dump() without circular imports - name: str """The unique identifier for this runnable component.""" description: str | None = None @@ -284,6 +304,15 @@ class Runnable(ABC, BaseModel): background_mode: Literal["auto", "always", "never"] = "never" """Background execution mode""" + offload_blocking: bool = False + """If True, plain sync handlers run in the default thread pool so blocking + code doesn't stall the event loop. Default False: sync handlers run inline + on the event loop, which avoids a threadpool round-trip per call. Set this + on tools whose sync handler blocks (network/disk I/O, heavy CPU) and that + must not serialize concurrent runs — or better, make the handler async. + Sync generator handlers always stream via the thread pool regardless of + this flag.""" + tracing_provider: Any = Field( default=TRACING_UNSET, description=( @@ -311,20 +340,30 @@ class Runnable(ABC, BaseModel): applicable to all Runnable types. """ - _path: str = PrivateAttr() - _is_orchestrator: bool = PrivateAttr() - _is_coroutine: bool = PrivateAttr() - _is_gen: bool = PrivateAttr() - _is_async_gen: bool = PrivateAttr() - _dependencies: list[str] = PrivateAttr(default_factory=list) - _default_fixed_params: dict[str, Any] = PrivateAttr(default_factory=dict) - _default_runtime_params: dict[str, dict[str, Any]] = PrivateAttr(default_factory=dict) - _pre_hook_is_coroutine: bool | None = PrivateAttr() - _pre_hook_dependencies: list[str] = PrivateAttr(default_factory=list) - _post_hook_is_coroutine: bool | None = PrivateAttr() - _post_hook_dependencies: list[str] = PrivateAttr(default_factory=list) - _log_events: set[str] = PrivateAttr() - _bg_tasks: dict[str, Any] = PrivateAttr(default_factory=dict) + # Workflow wiring. Declared as real (excluded-from-schema) fields rather than + # relying on extra="allow" attributes: declared fields live in the instance + # __dict__ and read ~8x faster than __pydantic_extra__ lookups, and these are + # read on every step execution. Set by Workflow.step(); None outside workflows. + previous_steps: Any = Field(default=None, exclude=True) + """Names of steps this step waits for (set by Workflow.step).""" + next_steps: Any = Field(default=None, exclude=True) + """Names of steps that depend on this step (set by Workflow.step).""" + previous_steps_kinds: Any = Field(default=None, exclude=True) + """Per-source edge kinds ('ordering' | 'when' | 'param') for introspection.""" + when: Any = Field(default=None, exclude=True) + """Optional {'callable', 'is_coroutine', ...} guard evaluated before the step runs.""" + + # NOTE — hot runtime attributes are plain instance attributes assigned in + # model_post_init, NOT pydantic PrivateAttr declarations. PrivateAttr reads + # route through BaseModel.__getattr__ (~20x slower than __dict__ lookup) and + # these are read multiple times on every call: + # _path, _is_orchestrator, _is_coroutine, _is_gen, _is_async_gen, + # _dependencies, _default_fixed_params, _default_runtime_params, + # _pre_hook_is_coroutine, _pre_hook_dependencies, + # _post_hook_is_coroutine, _post_hook_dependencies, + # _log_events, _bg_tasks + # Do not add class-level annotations for them — pydantic would turn any + # annotated underscore name back into a (slow) private attribute. @classmethod def _inspect_callable( @@ -414,17 +453,17 @@ def _inspect_callable( @field_validator("pre_hook", "post_hook") @classmethod - def _validate_hooks(cls, v: Any, info: ValidationInfo) -> Callable[[], Any] | None: - """Validate a hook, raise ValueError if invalid.""" + def _validate_hooks(cls, v: Any) -> Callable[[], Any] | None: + """Validate a hook, raise ValueError if invalid. + + Inspection *results* (is_coroutine, dependencies) are stored per instance + in model_post_init. They used to be written to ``cls`` here, which let two + instances of the same class with different sync/async hooks clobber each + other's flags. + """ if v is None: return v - inspect_result = cls._inspect_callable(v) - if info.field_name == "pre_hook": - cls._pre_hook_is_coroutine = inspect_result["is_coroutine"] - cls._pre_hook_dependencies = inspect_result["dependencies"] - elif info.field_name == "post_hook": - cls._post_hook_is_coroutine = inspect_result["is_coroutine"] - cls._post_hook_dependencies = inspect_result["dependencies"] + cls._inspect_callable(v) return v def _prepare_default_params(self, default_params: dict[str, Any]) -> None: @@ -443,6 +482,27 @@ def _prepare_default_params(self, default_params: dict[str, Any]) -> None: def model_post_init(self, __context: Any) -> None: """Initialize the Runnable after Pydantic model creation.""" + # Plain instance attributes (see NOTE above the field declarations). + self._dependencies: list[str] = [] + self._default_fixed_params: dict[str, Any] = {} + self._default_runtime_params: dict[str, dict[str, Any]] = {} + self._bg_tasks: dict[str, Any] = {} + self._blocking_warned: bool = False + if self.pre_hook is not None: + pre_hook_inspect = self._inspect_callable(self.pre_hook) + self._pre_hook_is_coroutine: bool | None = pre_hook_inspect["is_coroutine"] + self._pre_hook_dependencies: list[str] = pre_hook_inspect["dependencies"] + else: + self._pre_hook_is_coroutine = None + self._pre_hook_dependencies = [] + if self.post_hook is not None: + post_hook_inspect = self._inspect_callable(self.post_hook) + self._post_hook_is_coroutine: bool | None = post_hook_inspect["is_coroutine"] + self._post_hook_dependencies: list[str] = post_hook_inspect["dependencies"] + else: + self._post_hook_is_coroutine = None + self._post_hook_dependencies = [] + log_events = os.getenv("TIMBAL_LOG_EVENTS", "START,OUTPUT").split(",") self._log_events = set(event.strip() for event in log_events) self._prepare_default_params(self.default_params) @@ -796,17 +856,18 @@ def get_background_task(self, task_id: str) -> dict[str, Any]: return {"status": "running", "events": events, "name": task_info["name"], "input": task_info["input"]} async def _execute_runtime_callable(self, fn: Callable[..., Any], is_coroutine: bool) -> Any: - """Execute a runtime callable handling async context automatically.""" + """Execute a runtime callable handling async context automatically. + + Sync callables run inline on the event loop. These are param/`when` + lambdas and hooks — documented as cheap accessors over the run context + (e.g. ``step_span()``) — so a threadpool round-trip per evaluation is + pure overhead. Running inline also lets them see the live context vars + (the workflow sets ``parent_call_id`` right before evaluating them) + without a context copy. + """ if is_coroutine: return await fn() - else: - loop = asyncio.get_running_loop() - ctx = contextvars.copy_context() - - def fn_with_ctx(): - return ctx.run(fn) - - return await loop.run_in_executor(None, fn_with_ctx) + return fn() async def _execute_approval_callable(self, fn: Callable[..., Any], validated_input: dict[str, Any]) -> Any: """Execute an approval policy callable with matching validated input parameters.""" @@ -950,6 +1011,42 @@ async def _resolve_input_params(self, input: dict[str, Any] | None = None) -> di resolved.update(input) return resolved + async def _execute_simple(self, validated_input: dict[str, Any]) -> Any: + """Execute a non-streaming (plain sync or coroutine) handler and return its output. + + Fast path used by __call__ for handlers that cannot yield events — skips + the async-generator/tuple protocol of :meth:`_execute_handler` entirely. + Subclasses may override to add fallback behavior (see Tool's proxy path). + """ + if self._is_coroutine: + return await self.handler(**validated_input) + if self.offload_blocking: + loop = asyncio.get_running_loop() + ctx = contextvars.copy_context() + + def handler_func(): + return ctx.run(self.handler, **validated_input) + + return await loop.run_in_executor(None, handler_func) + # Sync handlers run inline on the event loop: while one runs, every + # other coroutine on this worker waits. Cheap handlers benefit (no + # threadpool hop); blocking ones degrade concurrent request latency, + # so flag them once with an actionable warning. + t0 = time.perf_counter() + try: + return self.handler(**validated_input) + finally: + elapsed_ms = (time.perf_counter() - t0) * 1e3 + if elapsed_ms >= _BLOCKING_HANDLER_WARN_MS and not self._blocking_warned: + self._blocking_warned = True + _get_logger().warning( + "Sync handler blocked the event loop; concurrent runs stall while it executes. " + "Make the handler async, or set offload_blocking=True to run it in a thread.", + runnable_path=self._path, + handler_elapsed_ms=round(elapsed_ms, 1), + threshold_ms=_BLOCKING_HANDLER_WARN_MS, + ) + async def _execute_handler( self, validated_input: dict[str, Any], run_context: Any, span: Any, event_queue: asyncio.Queue | None = None ) -> AsyncGenerator[tuple[Event | None, Any, Any], None]: @@ -963,22 +1060,15 @@ async def _execute_handler( output = None collector = None - if not self._is_async_gen and not self._is_coroutine: + if self._is_gen: loop = asyncio.get_running_loop() ctx = contextvars.copy_context() - if self._is_gen: - gen = self.handler(**validated_input) - async_gen = sync_to_async_gen(gen, loop, ctx) - else: - - def handler_func(): - return ctx.run(self.handler, **validated_input) - - output = await loop.run_in_executor(None, handler_func) - elif self._is_coroutine: - output = await self.handler(**validated_input) - else: + gen = self.handler(**validated_input) + async_gen = sync_to_async_gen(gen, loop, ctx) + elif self._is_async_gen: async_gen = self.handler(**validated_input) + else: + output = await self._execute_simple(validated_input) if async_gen: output = None @@ -1016,7 +1106,7 @@ def process_event(event): parent_call_id=span.parent_call_id, item=event, ) - if event.type in self._log_events: + if event.type in self._log_events and _events_logging_enabled(): _get_logger().info(event.type, **event.model_dump()) if event_queue: event_queue.put_nowait(event) @@ -1049,30 +1139,334 @@ def process_event(event): # Yield a final marker with the output and collector yield (None, output, collector) - @_timbal_collector_wrap - async def __call__(self, **kwargs: Any) -> AsyncGenerator[Event, None]: - """Execute the runnable with the given parameters. + async def _apply_approval_gate( + self, + approval_decision: ApprovalPolicyDecision, + validated_input: dict[str, Any], + span: Any, + run_context: Any, + ) -> tuple[bool, "ApprovalEvent | None", dict[str, Any]]: + """Run the human-approval gate for one invocation. + + All span status/output/metadata mutations for the gate happen here. + + Returns ``(proceed, approval_event, validated_input)``: + + - ``proceed=False, event=None`` — the gate ended the run (claimed by + another worker, cancelled, or denied); the caller just returns. + - ``proceed=False, event=ApprovalEvent`` — the gate is pending; the + caller yields the event and returns. + - ``proceed=True`` — approved; ``validated_input`` may carry the + human's edit-on-approve overrides. + """ + # ``approval_id`` MUST be derived from the unredacted input + # so the resume call (which carries the full input) lands + # on the same id as the original gate. + input_dump = await dump(validated_input) + approval_id = _approval_id_for(span.path, input_dump) + + # Compute the redacted view once and use it for every + # public surface. The unredacted ``validated_input`` is + # still what the handler sees on resume. + redacted_input = self._redact_validated_input(validated_input) + redaction_active = self.approval_redactor is not None or bool(self.approval_redact_keys) + if redaction_active: + span.input = redacted_input + span._input_dump = await dump(redacted_input) - This is the main entry point for executing a runnable. It handles: - - Parameter validation and merging with default_params - - Run context management and tracing setup - - Event streaming (StartEvent, DeltaEvents, OutputEvent) - - Error handling and cleanup - - Integration with the collectors system + try: + input_schema = self.format_params_model_schema() + except Exception: + input_schema = None + # Set by the agent on the START event when this gate fires inside a + # tool call (see Agent._safe_tool_dispatch). None for direct calls. + tool_call_id = span.metadata.get("tool_call_id") + span.metadata["approval"] = { + "id": approval_id, + "required": True, + "prompt": approval_decision.prompt, + "description": approval_decision.description, + "kind": approval_decision.kind, + "ui": approval_decision.ui, + "input_schema": input_schema, + "tool_call_id": tool_call_id, + "metadata": approval_decision.metadata, + "input": redacted_input, + } + approval_resolution = None + approval_cancel: Cancel | None = None + if approval_id in run_context._resume_values: + run_context._used_resume_ids.add(approval_id) + raw_resume = run_context._resume_values[approval_id] + # A Cancel value aborts the whole run rather than approving or + # denying. It is not a valid ApprovalResolution input, so it + # routes around coercion — but still flows through the durable + # claim below so a cancel and an approve racing on two workers + # can't both win. + if isinstance(raw_resume, Cancel): + approval_cancel = raw_resume + else: + approval_resolution = _coerce_approval_resolution(raw_resume) + if approval_resolution is not None and approval_resolution.is_expired(): + span.metadata["approval"]["expired"] = True + span.metadata["approval"]["expired_at"] = approval_resolution.expires_at + # Counter fires for the *expired* resolution. The gate + # then re-emits below, which adds a fresh :required tick. + run_context.update_usage("approvals:expired", 1) + approval_resolution = None + + # A decision OR a cancel claims the gate; the first claimer wins and + # any later duplicate (resolve or cancel) stops here. + if (approval_resolution is not None or approval_cancel is not None) and ( + run_context._tracing_provider is not None + ): + claimed = await run_context._tracing_provider.claim_approval( + str(run_context.parent_id) if run_context.parent_id else None, + approval_id, + str(run_context.id), + ) + if not claimed: + span.metadata["approval"]["claim"] = { + "claimed": False, + "parent_id": str(run_context.parent_id) if run_context.parent_id else None, + } + span.status = RunStatus( + code="cancelled", + reason="approval_already_claimed", + message="Approval was already claimed by another resume run.", + ) + span.output = { + "approval_id": approval_id, + "status": "approval_already_claimed", + } + span._output_dump = await dump(span.output) + return False, None, validated_input + + if approval_cancel is not None: + run_context.update_usage("approvals:cancelled", 1) + message = approval_cancel.reason or "Run cancelled by user." + span.metadata["approval"]["cancelled"] = True + span.status = RunStatus(code="cancelled", reason="cancelled", message=message) + span.output = {"approval_id": approval_id, "status": "cancelled", "reason": message} + span._output_dump = await dump(span.output) + return False, None, validated_input - The @collectable decorator wraps the returned async generator to add - a .collect() method for easy result collection. + if approval_resolution is None: + # Status/output MUST be set BEFORE the caller yields the event. + # If the consumer breaks the stream right after seeing the + # ApprovalEvent, GeneratorExit fires at the yield and we'd + # otherwise persist this span as 'interrupted'. + span.status = RunStatus( + code="cancelled", + reason="approval_required", + message="Approval required before runnable execution.", + ) + span.output = { + "approval_id": approval_id, + "status": "approval_required", + "prompt": approval_decision.prompt, + } + span._output_dump = await dump(span.output) + + approval_event = ApprovalEvent( + run_id=run_context.id, + parent_run_id=run_context.parent_id, + path=span.path, + call_id=span.call_id, + parent_call_id=span.parent_call_id, + t0=int(time.time() * 1000), + approval_id=approval_id, + runnable_path=span.path, + runnable_name=self.name, + runnable_type=self.metadata.get("type", self.__class__.__name__), + tool_call_id=tool_call_id, + input=redacted_input, + input_schema=input_schema, + prompt=approval_decision.prompt, + description=approval_decision.description, + kind=approval_decision.kind, + ui=approval_decision.ui, + metadata=approval_decision.metadata, + ) + run_context.update_usage("approvals:required", 1) + return False, approval_event, validated_input + + # Resolution found and not expired — capture the audit + # snapshot before deciding the gate's outcome. Typed fields + # are surfaced under ``resolution`` so trace consumers can + # query e.g. ``approval.resolution.approver_id`` directly. + span.metadata["approval"]["resolution"] = { + "approved": approval_resolution.approved, + "reason": approval_resolution.reason, + "approver_id": approval_resolution.approver_id, + "comment": approval_resolution.comment, + "decided_at": approval_resolution.decided_at, + "expires_at": approval_resolution.expires_at, + "override_input": approval_resolution.override_input, + "metadata": approval_resolution.metadata, + } + if not approval_resolution.approved: + run_context.update_usage("approvals:denied", 1) + span.status = RunStatus( + code="cancelled", + reason="approval_denied", + message=approval_resolution.reason or "Approval denied.", + ) + span.output = { + "approval_id": approval_id, + "status": "approval_denied", + "reason": approval_resolution.reason, + } + span._output_dump = await dump(span.output) + return False, None, validated_input + + run_context.update_usage("approvals:approved", 1) + + # Edit-on-approve: merge the human's overrides over the proposed + # input (override wins) and re-validate so the handler runs with + # the corrected, type-checked values. The audit snapshot above + # already recorded what was overridden. + if approval_resolution.override_input: + merged_input = {**validated_input, **approval_resolution.override_input} + validated_input = dict(self.params_model.model_validate(merged_input)) + # Reflect what actually runs on the public input surface (redacted). + span.input = self._redact_validated_input(validated_input) + span._input_dump = await dump(span.input) + span.metadata["approval"]["effective_input"] = span.input + + return True, None, validated_input + + async def _finalize_suspend(self, suspend_signal: Suspend, span: Any, run_context: Any) -> InteractionEvent: + """Record a ``suspend()`` pause on the span and build its InteractionEvent. + + The same durable resume rails as approvals (parent_id + tracing + provider) re-fire the handler with the supplied value on resume. + """ + span.status = RunStatus( + code="cancelled", + reason="input_required", + message="Input required to resume.", + ) + span.output = { + "suspension_id": suspend_signal.suspension_id, + "status": "input_required", + "kind": suspend_signal.kind, + "payload": suspend_signal.payload, + } + span._output_dump = await dump(span.output) + # Set by the agent on the START event when this suspension fires + # inside a tool call. None for direct (non-agent) calls. + tool_call_id = span.metadata.get("tool_call_id") + span.metadata["suspension"] = { + "id": suspend_signal.suspension_id, + "kind": suspend_signal.kind, + "payload": suspend_signal.payload, + "response_schema": suspend_signal.response_schema, + "tool_call_id": tool_call_id, + } + run_context.update_usage("suspends:required", 1) + return InteractionEvent( + run_id=run_context.id, + parent_run_id=run_context.parent_id, + path=span.path, + call_id=span.call_id, + parent_call_id=span.parent_call_id, + t0=int(time.time() * 1000), + interaction_id=suspend_signal.suspension_id, + kind=suspend_signal.kind, + runnable_path=span.path, + runnable_name=self.name, + runnable_type=self.metadata.get("type", self.__class__.__name__), + tool_call_id=tool_call_id, + payload=suspend_signal.payload, + response_schema=suspend_signal.response_schema, + ) + + def _spawn_background_task( + self, validated_input: dict[str, Any], input: dict[str, Any], run_context: Any, span: Any + ) -> dict[str, Any]: + """Spawn the handler as a background task registered on the parent runnable. + + Returns the ``{"task_id", "status"}`` placeholder output for the + launching span. The task streams its events into a queue polled via + :meth:`get_background_task` and records its real output on the span + when it completes. + """ + parent_span = run_context.parent_span() + if not parent_span: + raise ValueError("Parent span not found. Cannot run in background.") + task_id = "".join(secrets.choice(ALPHABET) for _ in range(6)) + event_queue = asyncio.Queue() + + async def _bg_handler_execution(): + output = None + try: + async for _, final_output, _handler_collector in self._execute_handler( + validated_input, run_context, span, event_queue + ): + if final_output is not None: + output = final_output + + # Post hook might modify the output, so we dump afterwards + span._output_dump = await dump(output) + span.output = output + _emit_default_tool_usage(self) + + set_parent_call_id(span.parent_call_id) + set_call_id(span.call_id) + if self.post_hook is not None: + await self._execute_runtime_callable(self.post_hook, self._post_hook_is_coroutine) + + except asyncio.CancelledError: + # Re-raise so asyncio marks the task as cancelled + raise + + task = asyncio.create_task(_bg_handler_execution(), context=contextvars.copy_context()) + + # Store task with event queue in parent runnable if available + parent_span.runnable._bg_tasks[task_id] = { + "task": task, + "event_queue": event_queue, + "name": self.name, + "input": input, + } + return {"task_id": task_id, "status": "running"} + + def __call__(self, **kwargs: Any) -> Any: + """Execute the runnable, returning a TimbalCollector over its event stream. + + This is the public entry point. The collector is the API boundary: it + supports ``async for`` iteration and ``.collect()``, and enriches the + final OutputEvent with pending approvals/interactions. Framework + internals (workflow steps, agent tools) iterate :meth:`_stream` + directly, skipping the per-event collector layer. Args: **kwargs: Runtime parameters for the runnable execution. Returns: - A BaseCollector that yields Events and provides collect() method + A TimbalCollector that yields Events and provides collect(). Raises: ValidationError: If input parameters don't match the params_model Exception: Any exception raised during handler execution (captured in OutputEvent) """ + global _TimbalCollector + if _TimbalCollector is None: + from ..collectors.impl.timbal import TimbalCollector + + _TimbalCollector = TimbalCollector + return _TimbalCollector(async_gen=self._stream(**kwargs)) + + async def _stream(self, **kwargs: Any) -> AsyncGenerator[Event, None]: + """Raw event stream for one runnable execution (internal entry point). + + Handles: + - Parameter validation and merging with default_params + - Run context management and tracing setup + - Event streaming (StartEvent, DeltaEvents, OutputEvent) + - Error handling and cleanup + """ t0 = int(time.time() * 1000) if self.background_mode == "auto": @@ -1091,6 +1485,13 @@ async def __call__(self, **kwargs: Any) -> AsyncGenerator[Event, None]: _parent_call_id = get_parent_call_id() _call_id = get_call_id() run_context = get_run_context() + # Entry snapshot for the finally block. A non-None entry call id means + # we're executing inside another runnable's call on this task; if we + # swap in a fresh RunContext below (top-level runnable invoked from a + # handler, e.g. an agent called inside a tool), the caller's context + # must be restored on exit or the swap leaks into the caller's run. + _entry_run_context = run_context + _entry_call_id = _call_id if run_context is None: run_context = RunContext(parent_id=explicit_parent_id, tracing_provider=self.tracing_provider) _parent_call_id = None @@ -1122,13 +1523,15 @@ async def __call__(self, **kwargs: Any) -> AsyncGenerator[Event, None]: ) _parent_call_id = None _call_id = None - await run_context.get_session() + # Session data is loaded once per run; nested calls see it already set. + if run_context._session_data is None: + await run_context.get_session() previous_resume_values = dict(run_context._resume_values) run_context._resume_values.update(resume_values) set_run_context(run_context) _new_parent_call_id = _call_id - _new_call_id: str = uuid7(as_type="str").replace("-", "") # type: ignore + _new_call_id: str = uuid7(as_type="hex") # type: ignore set_parent_call_id(_new_parent_call_id) set_call_id(_new_call_id) @@ -1149,9 +1552,11 @@ def _restore_context(): Between yields, another coroutine sharing the same asyncio Task may overwrite the context vars. Call this after every yield to reclaim ownership. Skips the writes if context is already correct - (the common single-consumer case). + (the common single-consumer case). The run-context check matters + independently of the call-id one: a nested top-level runnable may + have swapped in (and restored the ids around) a fresh RunContext. """ - if get_call_id() != _new_call_id: + if get_call_id() != _new_call_id or get_run_context() is not run_context: set_run_context(run_context) set_parent_call_id(_new_parent_call_id) set_call_id(_new_call_id) @@ -1172,7 +1577,7 @@ def _restore_context(): call_id=span.call_id, parent_call_id=span.parent_call_id, ) - if start_event.type in self._log_events: + if start_event.type in self._log_events and _events_logging_enabled(): _get_logger().info(start_event.type, **start_event.model_dump()) yield start_event _restore_context() @@ -1186,189 +1591,22 @@ def _restore_context(): # Pydantic model_validate() does not mutate the input dict validated_input = dict(self.params_model.model_validate(input)) - approval_decision = await self._resolve_approval_decision(validated_input) - if approval_decision.required: - # ``approval_id`` MUST be derived from the unredacted input - # so the resume call (which carries the full input) lands - # on the same id as the original gate. - input_dump = await dump(validated_input) - approval_id = _approval_id_for(span.path, input_dump) - - # Compute the redacted view once and use it for every - # public surface. The unredacted ``validated_input`` is - # still what the handler sees on resume. - redacted_input = self._redact_validated_input(validated_input) - redaction_active = ( - self.approval_redactor is not None or bool(self.approval_redact_keys) - ) - if redaction_active: - span.input = redacted_input - span._input_dump = await dump(redacted_input) - - try: - input_schema = self.format_params_model_schema() - except Exception: - input_schema = None - # Set by the agent on the START event when this gate fires inside a - # tool call (see Agent._safe_tool_dispatch). None for direct calls. - tool_call_id = span.metadata.get("tool_call_id") - span.metadata["approval"] = { - "id": approval_id, - "required": True, - "prompt": approval_decision.prompt, - "description": approval_decision.description, - "kind": approval_decision.kind, - "ui": approval_decision.ui, - "input_schema": input_schema, - "tool_call_id": tool_call_id, - "metadata": approval_decision.metadata, - "input": redacted_input, - } - approval_resolution = None - approval_cancel: Cancel | None = None - if approval_id in run_context._resume_values: - run_context._used_resume_ids.add(approval_id) - raw_resume = run_context._resume_values[approval_id] - # A Cancel value aborts the whole run rather than approving or - # denying. It is not a valid ApprovalResolution input, so it - # routes around coercion — but still flows through the durable - # claim below so a cancel and an approve racing on two workers - # can't both win. - if isinstance(raw_resume, Cancel): - approval_cancel = raw_resume - else: - approval_resolution = _coerce_approval_resolution(raw_resume) - if approval_resolution is not None and approval_resolution.is_expired(): - span.metadata["approval"]["expired"] = True - span.metadata["approval"]["expired_at"] = approval_resolution.expires_at - # Counter fires for the *expired* resolution. The gate - # then re-emits below, which adds a fresh :required tick. - run_context.update_usage("approvals:expired", 1) - approval_resolution = None - - # A decision OR a cancel claims the gate; the first claimer wins and - # any later duplicate (resolve or cancel) stops here. - if (approval_resolution is not None or approval_cancel is not None) and ( - run_context._tracing_provider is not None - ): - claimed = await run_context._tracing_provider.claim_approval( - str(run_context.parent_id) if run_context.parent_id else None, - approval_id, - str(run_context.id), + # Fast path: requires_approval=False (the default) skips the whole + # gate — no policy resolution, no ApprovalPolicyDecision construction. + if self.requires_approval is not False: + approval_decision = await self._resolve_approval_decision(validated_input) + if approval_decision.required: + proceed, approval_event, validated_input = await self._apply_approval_gate( + approval_decision, validated_input, span, run_context ) - if not claimed: - span.metadata["approval"]["claim"] = { - "claimed": False, - "parent_id": str(run_context.parent_id) if run_context.parent_id else None, - } - span.status = RunStatus( - code="cancelled", - reason="approval_already_claimed", - message="Approval was already claimed by another resume run.", - ) - span.output = { - "approval_id": approval_id, - "status": "approval_already_claimed", - } - span._output_dump = await dump(span.output) + if approval_event is not None: + if approval_event.type in self._log_events and _events_logging_enabled(): + _get_logger().info(approval_event.type, **approval_event.model_dump()) + yield approval_event + _restore_context() + if not proceed: return - if approval_cancel is not None: - run_context.update_usage("approvals:cancelled", 1) - message = approval_cancel.reason or "Run cancelled by user." - span.metadata["approval"]["cancelled"] = True - span.status = RunStatus(code="cancelled", reason="cancelled", message=message) - span.output = {"approval_id": approval_id, "status": "cancelled", "reason": message} - span._output_dump = await dump(span.output) - return - - if approval_resolution is None: - # Status/output MUST be set BEFORE the yield. If the - # consumer breaks the stream right after seeing the - # ApprovalEvent, GeneratorExit fires at the yield and - # we'd otherwise persist this span as 'interrupted'. - span.status = RunStatus( - code="cancelled", - reason="approval_required", - message="Approval required before runnable execution.", - ) - span.output = { - "approval_id": approval_id, - "status": "approval_required", - "prompt": approval_decision.prompt, - } - span._output_dump = await dump(span.output) - - approval_event = ApprovalEvent( - run_id=run_context.id, - parent_run_id=run_context.parent_id, - path=span.path, - call_id=span.call_id, - parent_call_id=span.parent_call_id, - t0=int(time.time() * 1000), - approval_id=approval_id, - runnable_path=span.path, - runnable_name=self.name, - runnable_type=self.metadata.get("type", self.__class__.__name__), - tool_call_id=tool_call_id, - input=redacted_input, - input_schema=input_schema, - prompt=approval_decision.prompt, - description=approval_decision.description, - kind=approval_decision.kind, - ui=approval_decision.ui, - metadata=approval_decision.metadata, - ) - run_context.update_usage("approvals:required", 1) - if approval_event.type in self._log_events: - _get_logger().info(approval_event.type, **approval_event.model_dump()) - yield approval_event - _restore_context() - return - - # Resolution found and not expired — capture the audit - # snapshot before deciding the gate's outcome. Typed fields - # are surfaced under ``resolution`` so trace consumers can - # query e.g. ``approval.resolution.approver_id`` directly. - span.metadata["approval"]["resolution"] = { - "approved": approval_resolution.approved, - "reason": approval_resolution.reason, - "approver_id": approval_resolution.approver_id, - "comment": approval_resolution.comment, - "decided_at": approval_resolution.decided_at, - "expires_at": approval_resolution.expires_at, - "override_input": approval_resolution.override_input, - "metadata": approval_resolution.metadata, - } - if not approval_resolution.approved: - run_context.update_usage("approvals:denied", 1) - span.status = RunStatus( - code="cancelled", - reason="approval_denied", - message=approval_resolution.reason or "Approval denied.", - ) - span.output = { - "approval_id": approval_id, - "status": "approval_denied", - "reason": approval_resolution.reason, - } - span._output_dump = await dump(span.output) - return - - run_context.update_usage("approvals:approved", 1) - - # Edit-on-approve: merge the human's overrides over the proposed - # input (override wins) and re-validate so the handler runs with - # the corrected, type-checked values. The audit snapshot above - # already recorded what was overridden. - if approval_resolution.override_input: - merged_input = {**validated_input, **approval_resolution.override_input} - validated_input = dict(self.params_model.model_validate(merged_input)) - # Reflect what actually runs on the public input surface (redacted). - span.input = self._redact_validated_input(validated_input) - span._input_dump = await dump(span.input) - span.metadata["approval"]["effective_input"] = span.input - # pre_hook runs only when we're actually going to execute the # handler. We deliberately defer it past the approval gate so # external side-effects don't fire on gated/denied attempts. @@ -1379,48 +1617,14 @@ def _restore_context(): # Background task if run_in_background: - parent_span = run_context.parent_span() - if not parent_span: - raise ValueError("Parent span not found. Cannot run in background.") - # task_id = uuid7(as_type="str").replace("-", "") - task_id = "".join(secrets.choice(ALPHABET) for _ in range(6)) - event_queue = asyncio.Queue() - - async def _bg_handler_execution(): - nonlocal output, collector - try: - async for _, final_output, handler_collector in self._execute_handler( - validated_input, run_context, span, event_queue - ): - if handler_collector is not None: - collector = handler_collector - if final_output is not None: - output = final_output - - # Post hook might modify the output, so we dump afterwards - span._output_dump = await dump(output) - span.output = output - _emit_default_tool_usage(self) - - set_parent_call_id(_new_parent_call_id) - set_call_id(_new_call_id) - if self.post_hook is not None: - await self._execute_runtime_callable(self.post_hook, self._post_hook_is_coroutine) - - except asyncio.CancelledError: - # Re-raise so asyncio marks the task as cancelled - raise - - task = asyncio.create_task(_bg_handler_execution(), context=contextvars.copy_context()) - - # Store task with event queue in parent runnable if available - parent_span.runnable._bg_tasks[task_id] = { - "task": task, - "event_queue": event_queue, - "name": self.name, - "input": input, - } - output = {"task_id": task_id, "status": "running"} + output = self._spawn_background_task(validated_input, input, run_context, span) + elif not self._is_async_gen and not self._is_gen: + # Fast path: plain sync/coroutine handlers cannot yield events, + # so skip the async-generator/tuple protocol entirely. + try: + output = await self._execute_simple(validated_input) + except Suspend as susp: + suspend_signal = susp else: # Iterate over events from handler and yield them try: @@ -1455,46 +1659,8 @@ async def _bg_handler_execution(): suspend_signal = susp if suspend_signal is not None: - span.status = RunStatus( - code="cancelled", - reason="input_required", - message="Input required to resume.", - ) - span.output = { - "suspension_id": suspend_signal.suspension_id, - "status": "input_required", - "kind": suspend_signal.kind, - "payload": suspend_signal.payload, - } - span._output_dump = await dump(span.output) - # Set by the agent on the START event when this suspension fires - # inside a tool call. None for direct (non-agent) calls. - tool_call_id = span.metadata.get("tool_call_id") - span.metadata["suspension"] = { - "id": suspend_signal.suspension_id, - "kind": suspend_signal.kind, - "payload": suspend_signal.payload, - "response_schema": suspend_signal.response_schema, - "tool_call_id": tool_call_id, - } - run_context.update_usage("suspends:required", 1) - interaction_event = InteractionEvent( - run_id=run_context.id, - parent_run_id=run_context.parent_id, - path=span.path, - call_id=span.call_id, - parent_call_id=span.parent_call_id, - t0=int(time.time() * 1000), - interaction_id=suspend_signal.suspension_id, - kind=suspend_signal.kind, - runnable_path=span.path, - runnable_name=self.name, - runnable_type=self.metadata.get("type", self.__class__.__name__), - tool_call_id=tool_call_id, - payload=suspend_signal.payload, - response_schema=suspend_signal.response_schema, - ) - if interaction_event.type in self._log_events: + interaction_event = await self._finalize_suspend(suspend_signal, span, run_context) + if interaction_event.type in self._log_events and _events_logging_enabled(): _get_logger().info(interaction_event.type, **interaction_event.model_dump()) yield interaction_event _restore_context() @@ -1524,6 +1690,9 @@ async def _bg_handler_execution(): set_call_id(_new_call_id) if self.post_hook is not None and not run_in_background: await self._execute_runtime_callable(self.post_hook, self._post_hook_is_coroutine) + # Hooks may mutate message content in place; drop any cached + # dumps on the output so the re-dump below sees the changes. + invalidate_message_dump_caches(span.output) # Post hook might modify the output, so we dump afterwards span._output_dump = await dump(span.output) @@ -1691,7 +1860,16 @@ async def _bg_handler_execution(): run_context._resume_values = previous_resume_values set_parent_call_id(_parent_call_id) set_call_id(_call_id) - if output_event.type in self._log_events: + if _entry_call_id is not None: + # Nested invocation: restore the caller's run context. A + # top-level runnable invoked from a handler may have swapped in + # a fresh RunContext above; with direct in-task iteration + # (agent tool fast path, linear workflow steps) that swap would + # otherwise leak into the caller's run. Top-level invocations + # (entry call id None) deliberately leave their context set so + # sequential same-task runs chain sessions implicitly. + set_run_context(_entry_run_context) + if output_event.type in self._log_events and _events_logging_enabled(): _get_logger().info(output_event.type, **output_event.model_dump()) if not _generator_closed: yield output_event diff --git a/python/timbal/core/test_model.py b/python/timbal/core/test_model.py index 0c39545e..9252cfbf 100644 --- a/python/timbal/core/test_model.py +++ b/python/timbal/core/test_model.py @@ -112,9 +112,18 @@ async def stream(self, messages: list, **_kwargs: Any) -> AsyncGenerator: response = raw # Approximate token counts (1 token ≈ 4 chars) for UsageLimits compatibility. + # Fast path for TextContent (the bulk of long histories): stringifying + # whole pydantic content models is O(model repr) per item per call. + # Non-text blocks (tool results etc.) keep the str() estimate — memory + # compaction tests rely on large tool results driving usage up. + def _content_chars(c: Any) -> int: + if isinstance(c, TextContent): + return len(c.text) + return len(str(c)) + run_context = get_or_create_run_context() - input_tokens = max(1, sum(len(str(c)) for m in messages for c in m.content) // 4) - output_tokens = max(1, sum(len(str(c)) for c in response.content) // 4) + input_tokens = max(1, sum(_content_chars(c) for m in messages for c in m.content) // 4) + output_tokens = max(1, sum(_content_chars(c) for c in response.content) // 4) run_context.update_usage("test/model:input_text_tokens", input_tokens) run_context.update_usage("test/model:output_text_tokens", output_tokens) diff --git a/python/timbal/core/tool.py b/python/timbal/core/tool.py index dd1c44a4..980adaba 100644 --- a/python/timbal/core/tool.py +++ b/python/timbal/core/tool.py @@ -121,6 +121,38 @@ def model_post_init(self, __context: Any) -> None: self._is_async_gen = inspect_result["is_async_gen"] self._dependencies = inspect_result["dependencies"] + @override + async def _execute_simple(self, validated_input: dict[str, Any]) -> Any: + """Execute a non-streaming handler locally; on missing credentials, run via + platform tool proxy. Same fallback contract as :meth:`_execute_handler` + (which covers streaming handlers) — see its docstring for the rationale. + """ + cred_error: CredentialNotAvailable | None = None + try: + return await super()._execute_simple(validated_input) + except CredentialNotAvailable as e: + # Captured and handled outside the except block so proxy failures + # don't implicitly chain the credential error (see _execute_handler). + cred_error = e + + from ..state import get_run_context + + try: + return await execute_tool_proxy(self.name, validated_input) + except ToolProxyUnavailable: + run_context = get_run_context() + if run_context is None or run_context.platform_config is None: + # Local run without platform config — fall back to credential error. + raise cred_error from None + raise + except PlatformError as proxy_error: + # 403 is what the platform returns when no proxy is available for this + # tool (no service-account credentials configured). 404/501 are kept as + # fallbacks. In all these cases, surface the actionable credential error. + if proxy_error.status_code in (403, 404, 501): + raise cred_error from None + raise + @override async def _execute_handler( self, @@ -141,6 +173,11 @@ async def _execute_handler( ``TIMBAL_API_KEY`` without ``TIMBAL_ORG_ID``), propagate ``ToolProxyUnavailable`` so the user fixes platform setup rather than local provider credentials. + + Note: non-streaming handlers reached through this method (background + mode) already get proxy fallback inside ``_execute_simple``; the catch + here can then only re-fire on the re-raised credential error, causing + at most one redundant proxy attempt on an already-failing path. """ cred_error: CredentialNotAvailable | None = None try: diff --git a/python/timbal/core/workflow.py b/python/timbal/core/workflow.py index 86431485..390a4d7e 100644 --- a/python/timbal/core/workflow.py +++ b/python/timbal/core/workflow.py @@ -12,7 +12,7 @@ from typing_extensions import override import structlog -from pydantic import BaseModel, ConfigDict, PrivateAttr, computed_field, create_model +from pydantic import BaseModel, ConfigDict, computed_field, create_model from ..errors import InterruptError, PauseRequired, RunCancelled, SpanNotFound, WorkflowStepError from ..state import get_call_id, get_parent_call_id, set_parent_call_id @@ -30,12 +30,16 @@ class StepState(Enum): class StepStatus: - __slots__ = ("state", "done", "error") + __slots__ = ("state", "done", "error", "signal") def __init__(self) -> None: self.state: StepState = StepState.PENDING self.done: asyncio.Event = asyncio.Event() self.error: dict[str, Any] | None = None + self.signal: BaseException | None = None + """Deferred outcome signal recorded by _run_step: PauseRequired, + RunCancelled, or the exception raised while streaming the step. + Aggregated by handler() after the step finishes.""" logger = structlog.get_logger("timbal.core.workflow") @@ -44,10 +48,11 @@ def __init__(self) -> None: class Workflow(Runnable): """Orchestrates execution of multiple steps in a DAG with automatic dependency linking.""" - _steps: dict[str, Runnable] = PrivateAttr(default_factory=dict) - def model_post_init(self, __context: Any) -> None: super().model_post_init(__context) + # Plain instance attributes (not PrivateAttr) — hot reads on every run. + self._steps: dict[str, Runnable] = {} + self._is_linear: bool = True self._path = self.name self._is_orchestrator = True self._is_coroutine = False @@ -192,34 +197,44 @@ def step( logger.info("Linking steps", previous_step=dep, next_step=runnable.name) self._link(dep, runnable.name) + # A workflow is a linear chain when each step depends exactly on its + # predecessor (and the first on nothing). handler() then skips the + # task/queue fan-in machinery entirely. + steps = list(self._steps.values()) + self._is_linear = all( + (not s.previous_steps) if i == 0 else s.previous_steps == {steps[i - 1].name} + for i, s in enumerate(steps) + ) + return self - async def _enqueue_step_events( + async def _run_step( self, step: Runnable, - queue: asyncio.Queue, statuses: dict[str, StepStatus], **kwargs: Any, - ) -> None: - """Execute a single workflow step and enqueue its events to the shared queue. - - Sentinel contract: this coroutine MUST push exactly one sentinel value to - ``queue`` on every exit path (None, an Exception, or a PauseRequired). - The consumer in :meth:`handler` decrements ``remaining`` on each sentinel, - and a missed sentinel hangs the consumer on ``queue.get()`` forever. The - outer try/finally below guarantees this even when a ``BaseException`` - subclass (e.g. a user-defined ``when`` callable raising ``BaseException``, - or task-level ``CancelledError``) leaks past ``except Exception``. + ) -> AsyncGenerator[Any, None]: + """Execute a single workflow step and yield its events. + + The step's outcome is recorded on ``statuses[step.name]``: + ``state``/``error`` for regular completion/failure/skip, and ``signal`` + for deferred control flow (PauseRequired, RunCancelled, or the raised + exception). ``done`` is ALWAYS set on exit so dependents never hang. + + Shared by both execution modes in :meth:`handler` — the sequential + fast path iterates it directly; the concurrent path drives it from a + task via :meth:`_enqueue_step_events`. """ status = statuses[step.name] - # Value pushed to the queue exactly once when this coroutine exits. - # Defaults to None (= regular completion); branches that need a different - # signal (Exception, PauseRequired) overwrite it before returning. - sentinel: Any = None try: - # Await for the completion of all ancestors - await asyncio.gather(*[statuses[step_name].done.wait() for step_name in step.previous_steps]) + # Await for the completion of all ancestors. Sequential awaits are + # equivalent to gather() for waiting on ALL events, without a task + # per dependency; the is_set() guard skips already-completed ones. + for step_name in step.previous_steps: + dep_done = statuses[step_name].done + if not dep_done.is_set(): + await dep_done.wait() # This serves multiple purposes. # - It ensures that the step is not executed multiple times. # - It allows the step to be skipped from other steps, e.g. if a previous step failed. @@ -270,8 +285,11 @@ async def _enqueue_step_events( status.state = StepState.RUNNING try: - async for event in step(**resolved_input): - await queue.put(event) + # Iterate the raw stream: the TimbalCollector wrapper is only + # needed at the public API boundary (.collect(), pending-gate + # enrichment); a per-event collector layer here is pure overhead. + async for event in step._stream(**resolved_input): + yield event if ( isinstance(event, OutputEvent) and event.status.code == "cancelled" @@ -279,7 +297,7 @@ async def _enqueue_step_events( ): logger.info(f"Step {step.name} paused ({event.status.reason}).") status.state = StepState.FAILED - sentinel = PauseRequired(event) + status.signal = PauseRequired(event) return if ( isinstance(event, OutputEvent) @@ -290,7 +308,7 @@ async def _enqueue_step_events( # whole workflow run rather than continuing other steps. logger.info(f"Step {step.name} cancelled by user.") status.state = StepState.FAILED - sentinel = RunCancelled(event.status.message or "Run cancelled by user.") + status.signal = RunCancelled(event.status.message or "Run cancelled by user.") return if isinstance(event, OutputEvent) and event.error is not None: logger.info(f"Step {step.name} completed with error.") @@ -306,20 +324,19 @@ async def _enqueue_step_events( "message": str(e), "traceback": traceback.format_exc(), } - sentinel = e + status.signal = e return finally: status.done.set() except BaseException as e: # Catch BaseException subclasses that bypass the inner `except Exception` - # (custom BaseException from user `when`/resolver callables, task-level - # CancelledError, etc.). Without this branch the outer `finally` still - # guarantees the sentinel, but flagging the step as FAILED keeps - # downstream-step state consistent. Re-raise so workflow cleanup - # (gather(..., return_exceptions=True) in handler's finally) proceeds. + # (custom BaseException from user `when`/resolver callables, consumer + # GeneratorExit, task-level CancelledError, etc.). Flagging the step + # as FAILED and setting `done` keeps downstream-step state consistent. + # Re-raise so the consumer (or task cleanup) proceeds. logger.warning( - "Step %s exited via BaseException %s; ensuring sentinel is enqueued.", + "Step %s exited via BaseException %s.", step.name, type(e).__name__, ) @@ -329,17 +346,44 @@ async def _enqueue_step_events( raise finally: - # GUARANTEE the consumer in handler() always sees a sentinel for this - # step. put_nowait is safe because the queue is unbounded; using a - # non-awaiting call here also avoids re-entering the event loop during - # exception unwinding. + if not status.done.is_set(): + status.done.set() + + async def _enqueue_step_events( + self, + step: Runnable, + queue: asyncio.Queue, + statuses: dict[str, StepStatus], + **kwargs: Any, + ) -> None: + """Drive one step for concurrent execution, forwarding its events to the queue. + + Sentinel contract: exactly one sentinel — the step's own StepStatus — + is pushed on every exit path. The consumer in :meth:`handler` + decrements ``remaining`` per StepStatus and reads its ``signal``; a + missed sentinel would hang the consumer on ``queue.get()`` forever, + so the finally guarantees it even for BaseException exits. + """ + status = statuses[step.name] + try: + async for event in self._run_step(step, statuses, **kwargs): + # put_nowait: the queue is unbounded, so put() never suspends — + # awaiting it is pure coroutine overhead per event. + queue.put_nowait(event) + finally: try: - queue.put_nowait(sentinel) + queue.put_nowait(status) except Exception: logger.exception("Failed to enqueue sentinel for step %s", step.name) async def handler(self, **kwargs: Any) -> AsyncGenerator[Any, None]: - """Execute all steps concurrently, respecting dependencies. + """Execute all steps, respecting dependencies. + + Linear chains (each step depends exactly on its predecessor — the + common sequential pipeline) run in a direct loop with no tasks or + queue: concurrency is impossible, so the fan-in machinery is pure + overhead. Everything else runs one task per step multiplexed + through a queue. When multiple parallel steps pause (approval gate or suspend()), every pause is drained (so each step emits its OutputEvent + Approval/ @@ -349,55 +393,88 @@ async def handler(self, **kwargs: Any) -> AsyncGenerator[Any, None]: tool-multiplexing behaviour and prevents the first pause from cancelling later ones. """ - queue = asyncio.Queue() statuses = {step_name: StepStatus() for step_name in self._steps.keys()} - tasks = [ - asyncio.create_task(self._enqueue_step_events(step, queue, statuses, **kwargs)) - for step in self._steps.values() - ] - first_pending_pause: PauseRequired | None = None first_pending_exception: Exception | None = None - try: - remaining = len(tasks) - while remaining > 0: - event = await queue.get() - if isinstance(event, InterruptError): - raise event - if isinstance(event, PauseRequired): - if first_pending_pause is None: - first_pending_pause = event - remaining -= 1 - continue - if isinstance(event, Exception): - if first_pending_exception is None: - first_pending_exception = event - remaining -= 1 - continue - if event is None: - remaining -= 1 - else: - yield event - # A pause (approval or suspend) takes precedence over a step error in - # the surfaced status; both ride the same durable resume rails. - if first_pending_pause is not None: - raise first_pending_pause - if first_pending_exception is not None: - raise first_pending_exception - failed_steps = sorted( - (name, step_status) - for name, step_status in statuses.items() - if step_status.state == StepState.FAILED - ) - if failed_steps: - step_name, step_status = failed_steps[0] - raise WorkflowStepError(step_name, step_status.error) - except (asyncio.CancelledError, InterruptError): - raise - finally: - for task in tasks: - if not task.done(): - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) + def _record_signal(signal: BaseException | None) -> None: + nonlocal first_pending_pause, first_pending_exception + if isinstance(signal, InterruptError): + raise signal + if isinstance(signal, PauseRequired): + if first_pending_pause is None: + first_pending_pause = signal + elif isinstance(signal, Exception): + if first_pending_exception is None: + first_pending_exception = signal + + if self._is_linear: + # Sequential fast path — insertion order is topological (links can + # only point at already-registered steps). + current_task = asyncio.current_task() + + def _cancel_pending() -> bool: + # A cancelled step records 'interrupted' on its own span and + # swallows the CancelledError. In concurrent mode the workflow + # still sees the cancellation at queue.get() (and never yields + # the interrupted step's final event); here we share the task + # with the step, so detect the still-pending cancel request and + # re-raise before forwarding post-cancel events. + return current_task is not None and current_task.cancelling() + + for step in self._steps.values(): + try: + async for event in self._run_step(step, statuses, **kwargs): + if _cancel_pending(): + raise asyncio.CancelledError + yield event + except (asyncio.CancelledError, GeneratorExit, InterruptError): + raise + except BaseException: # noqa: BLE001 + # Parity with concurrent mode, where a BaseException from a + # user `when`/resolver callable dies inside the step task + # (gather(..., return_exceptions=True)) and the workflow + # surfaces the failure via failed_steps below. _run_step + # already marked the step FAILED and logged. + pass + if _cancel_pending(): + raise asyncio.CancelledError + _record_signal(statuses[step.name].signal) + else: + queue = asyncio.Queue() + tasks = [ + asyncio.create_task(self._enqueue_step_events(step, queue, statuses, **kwargs)) + for step in self._steps.values() + ] + try: + remaining = len(tasks) + while remaining > 0: + item = await queue.get() + if isinstance(item, StepStatus): + remaining -= 1 + _record_signal(item.signal) + else: + yield item + except (asyncio.CancelledError, InterruptError): + raise + finally: + for task in tasks: + if not task.done(): + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + # A pause (approval or suspend) takes precedence over a step error in + # the surfaced status; both ride the same durable resume rails. + if first_pending_pause is not None: + raise first_pending_pause + if first_pending_exception is not None: + raise first_pending_exception + failed_steps = sorted( + (name, step_status) + for name, step_status in statuses.items() + if step_status.state == StepState.FAILED + ) + if failed_steps: + step_name, step_status = failed_steps[0] + raise WorkflowStepError(step_name, step_status.error) diff --git a/python/timbal/evals/validators/_spans.py b/python/timbal/evals/validators/_spans.py new file mode 100644 index 00000000..756d8fc8 --- /dev/null +++ b/python/timbal/evals/validators/_spans.py @@ -0,0 +1,47 @@ +"""Shared span-timing helpers for seq! and parallel! validators.""" + +from ...state.tracing.span import Span + + +def get_span_name(span_path: str) -> str: + """Extract the span name from a full path.""" + return span_path.rsplit(".", 1)[-1] if "." in span_path else span_path + + +def spans_overlap(span_a: Span, span_b: Span, tolerance_ms: int = 0) -> bool: + """Check if two spans have overlapping time ranges. + + Two spans overlap if they share any point in time (within tolerance). + Using <= to handle edge case where spans have identical start/end times. + If either span hasn't completed, a large value is used for its end time. + """ + t1_a = span_a.t1 if span_a.t1 is not None else float("inf") + t1_b = span_b.t1 if span_b.t1 is not None else float("inf") + return span_a.t0 <= t1_b + tolerance_ms and span_b.t0 <= t1_a + tolerance_ms + + +def validate_parallel_spans(spans: list[Span], tolerance_ms: int = 0, *, label: str = "") -> tuple[bool, str]: + """Check if all spans ran in parallel (all pairs overlap). + + *label* is inserted into the error message after the span names (e.g. + ``" in parallel!"`` for seq!'s nested parallel patterns). + + Returns: + Tuple of (all_parallel, error_message) + """ + if len(spans) < 2: + return True, "" + + for i, span_a in enumerate(spans): + for span_b in spans[i + 1 :]: + if not spans_overlap(span_a, span_b, tolerance_ms): + name_a = get_span_name(span_a.path) + name_b = get_span_name(span_b.path) + t1_a_str = f"{span_a.t1}" if span_a.t1 else "running" + t1_b_str = f"{span_b.t1}" if span_b.t1 else "running" + return False, ( + f"spans '{name_a}' and '{name_b}'{label} did not run in parallel. " + f"'{name_a}': {span_a.t0}-{t1_a_str}, " + f"'{name_b}': {span_b.t0}-{t1_b_str}" + ) + return True, "" diff --git a/python/timbal/evals/validators/base.py b/python/timbal/evals/validators/base.py index 801ef7fa..681b5f37 100644 --- a/python/timbal/evals/validators/base.py +++ b/python/timbal/evals/validators/base.py @@ -118,6 +118,30 @@ def extract_options_from_value(cls, data: Any) -> Any: return data + def _normalize_bool_value(self) -> None: + """Normalize ``value`` to a boolean expectation (default True).""" + self.value = True if self.value is None else bool(self.value) + + def _resolve_expected_str(self, ctx: ValidationContext, kind: str) -> str | None: + """Resolve the target to a string for boolean-expectation validators. + + Returns None when the value is non-string and the expectation is False + (nothing further to check). Raises when a string was expected. + """ + from ...types.message import Message + from ..utils import resolve_target + + _, actual_value = resolve_target(ctx.trace, self.target, self.path_key) + + if isinstance(actual_value, Message): + actual_value = actual_value.collect_text() + + if not isinstance(actual_value, str): + if self.value: + raise AssertionError(f"expected {kind} string, got {type(actual_value).__name__}") + return None + return actual_value + def apply_transform(self, value: str) -> str: """Apply configured transforms to a value. diff --git a/python/timbal/evals/validators/email.py b/python/timbal/evals/validators/email.py index 2394348b..5da049a9 100644 --- a/python/timbal/evals/validators/email.py +++ b/python/timbal/evals/validators/email.py @@ -2,7 +2,6 @@ from pydantic import BaseModel, EmailStr, ValidationError, model_validator -from ...types.message import Message from .base import BaseValidator from .context import ValidationContext @@ -20,11 +19,7 @@ class EmailValidator(BaseValidator): @model_validator(mode="after") def validate_value(self) -> "EmailValidator": - # Normalize value to boolean (default True if not provided) - if self.value is None: - self.value = True - else: - self.value = bool(self.value) + self._normalize_bool_value() return self async def __call__(self, ctx: ValidationContext) -> None: @@ -35,24 +30,14 @@ async def __call__(self, ctx: ValidationContext) -> None: Raises: AssertionError: If email validity doesn't match expected state. """ - from ..utils import resolve_target - - _, actual_value = resolve_target(ctx.trace, self.target, self.path_key) - - if isinstance(actual_value, Message): - actual_value = actual_value.collect_text() - - if not isinstance(actual_value, str): - if self.value: - raise AssertionError(f"expected email string, got {type(actual_value).__name__}") - else: - # Non-string is not an email, which is what we wanted - return + actual_value = self._resolve_expected_str(ctx, "email") + if actual_value is None: + # Non-string is not an email, which is what we wanted + return actual_value = actual_value.strip() # Use Pydantic model to validate email - is_valid_email = False try: _EmailModel(email=actual_value) is_valid_email = True diff --git a/python/timbal/evals/validators/json.py b/python/timbal/evals/validators/json.py index b500ad2b..7a134ecb 100644 --- a/python/timbal/evals/validators/json.py +++ b/python/timbal/evals/validators/json.py @@ -2,7 +2,6 @@ from pydantic import model_validator -from ...types.message import Message from .base import BaseValidator from .context import ValidationContext @@ -14,11 +13,7 @@ class JsonValidator(BaseValidator): @model_validator(mode="after") def validate_value(self) -> "JsonValidator": - # Normalize value to boolean (default True if not provided) - if self.value is None: - self.value = True - else: - self.value = bool(self.value) + self._normalize_bool_value() return self async def __call__(self, ctx: ValidationContext) -> None: @@ -29,19 +24,10 @@ async def __call__(self, ctx: ValidationContext) -> None: Raises: AssertionError: If JSON validity doesn't match expected state. """ - from ..utils import resolve_target - - _, actual_value = resolve_target(ctx.trace, self.target, self.path_key) - - if isinstance(actual_value, Message): - actual_value = actual_value.collect_text() - - if not isinstance(actual_value, str): - if self.value: - raise AssertionError(f"expected JSON string, got {type(actual_value).__name__}") - else: - # Non-string is not JSON, which is what we wanted - return + actual_value = self._resolve_expected_str(ctx, "JSON") + if actual_value is None: + # Non-string is not JSON, which is what we wanted + return # Strip markdown code blocks (```json ... ``` or ``` ... ```) import re @@ -55,7 +41,6 @@ async def __call__(self, ctx: ValidationContext) -> None: # Try to parse as JSON import json - is_valid_json = False try: json.loads(actual_value) is_valid_json = True diff --git a/python/timbal/evals/validators/parallel.py b/python/timbal/evals/validators/parallel.py index 45cd8d66..62b71788 100644 --- a/python/timbal/evals/validators/parallel.py +++ b/python/timbal/evals/validators/parallel.py @@ -1,6 +1,7 @@ from typing import Literal from ...state.tracing.span import Span +from ._spans import get_span_name, validate_parallel_spans from .base import BaseValidator from .context import ValidationContext @@ -73,53 +74,6 @@ def _parse_value(self) -> tuple[list[str], int]: return expected_spans, tolerance_ms - def _get_span_name(self, span_path: str) -> str: - """Extract the span name from a full path.""" - return span_path.rsplit(".", 1)[-1] if "." in span_path else span_path - - def _spans_overlap(self, span_a: Span, span_b: Span, tolerance_ms: int = 0) -> bool: - """Check if two spans have overlapping time ranges. - - Two spans overlap if they share any point in time (within tolerance). - Using <= to handle edge case where spans have identical start/end times. - - Args: - span_a: First span - span_b: Second span - tolerance_ms: Tolerance in milliseconds - spans can have this much gap and still be considered parallel - """ - # If either span hasn't completed, use a large value for t1 - t1_a = span_a.t1 if span_a.t1 is not None else float("inf") - t1_b = span_b.t1 if span_b.t1 is not None else float("inf") - - # Add tolerance to the end times - return span_a.t0 <= t1_b + tolerance_ms and span_b.t0 <= t1_a + tolerance_ms - - def _all_spans_parallel(self, spans: list[Span], tolerance_ms: int = 0) -> tuple[bool, str]: - """Check if all spans ran in parallel (all pairs overlap). - - Returns: - Tuple of (all_parallel, error_message) - """ - if len(spans) < 2: - return True, "" - - # Check all pairs - for i, span_a in enumerate(spans): - for span_b in spans[i + 1 :]: - if not self._spans_overlap(span_a, span_b, tolerance_ms): - name_a = self._get_span_name(span_a.path) - name_b = self._get_span_name(span_b.path) - t1_a_str = f"{span_a.t1}" if span_a.t1 else "running" - t1_b_str = f"{span_b.t1}" if span_b.t1 else "running" - return False, ( - f"spans '{name_a}' and '{name_b}' did not run in parallel. " - f"'{name_a}': {span_a.t0}-{t1_a_str}, " - f"'{name_b}': {span_b.t0}-{t1_b_str}" - ) - - return True, "" - async def __call__(self, ctx: ValidationContext) -> None: """Check if all expected spans exist and ran in parallel. @@ -137,7 +91,7 @@ async def __call__(self, ctx: ValidationContext) -> None: # Build a map of span name to span span_map: dict[str, Span] = {} for span in all_spans: - name = self._get_span_name(span.path) + name = get_span_name(span.path) span_map[name] = span # Check that all expected spans exist @@ -149,6 +103,6 @@ async def __call__(self, ctx: ValidationContext) -> None: matched_spans = [span_map[name] for name in expected_spans] # Check that all matched spans ran in parallel - all_parallel, error_msg = self._all_spans_parallel(matched_spans, tolerance_ms) + all_parallel, error_msg = validate_parallel_spans(matched_spans, tolerance_ms) if not all_parallel: raise AssertionError(error_msg) diff --git a/python/timbal/evals/validators/seq.py b/python/timbal/evals/validators/seq.py index 440848f2..d2fb423c 100644 --- a/python/timbal/evals/validators/seq.py +++ b/python/timbal/evals/validators/seq.py @@ -4,6 +4,7 @@ from typing import Literal from ...state.tracing.span import Span +from ._spans import get_span_name, validate_parallel_spans from .base import BaseValidator from .context import ValidationContext @@ -167,11 +168,6 @@ def _parse_patterns(self) -> list[str | AnyPattern | ParallelPattern]: return patterns - def _get_span_name(self, span_path: str) -> str: - """Extract the span name from a full path.""" - # Path is like "agent.tool.subtool", we want "subtool" - return span_path.rsplit(".", 1)[-1] if "." in span_path else span_path - def _dp_match(self, patterns: list[str | AnyPattern | ParallelPattern], span_names: list[str]) -> tuple[bool, str]: """Use dynamic programming to match patterns against span names. @@ -262,40 +258,6 @@ def _dp_match(self, patterns: list[str | AnyPattern | ParallelPattern], span_nam f"expected sequence ({' -> '.join(pattern_strs)}) but got ({' -> '.join(span_names)})", ) - def _spans_overlap(self, span_a: Span, span_b: Span, tolerance_ms: int = 0) -> bool: - """Check if two spans have overlapping time ranges. - - Two spans overlap if they share any point in time (within tolerance). - Using <= to handle edge case where spans have identical start/end times. - - Args: - span_a: First span - span_b: Second span - tolerance_ms: Tolerance in milliseconds - spans can have this much gap and still be considered parallel - """ - t1_a = span_a.t1 if span_a.t1 is not None else float("inf") - t1_b = span_b.t1 if span_b.t1 is not None else float("inf") - return span_a.t0 <= t1_b + tolerance_ms and span_b.t0 <= t1_a + tolerance_ms - - def _validate_parallel_spans(self, spans: list[Span], tolerance_ms: int = 0) -> tuple[bool, str]: - """Check if all spans ran in parallel (all pairs overlap).""" - if len(spans) < 2: - return True, "" - - for i, span_a in enumerate(spans): - for span_b in spans[i + 1 :]: - if not self._spans_overlap(span_a, span_b, tolerance_ms): - name_a = self._get_span_name(span_a.path) - name_b = self._get_span_name(span_b.path) - t1_a_str = f"{span_a.t1}" if span_a.t1 else "running" - t1_b_str = f"{span_b.t1}" if span_b.t1 else "running" - return False, ( - f"spans '{name_a}' and '{name_b}' in parallel! did not run in parallel. " - f"'{name_a}': {span_a.t0}-{t1_a_str}, " - f"'{name_b}': {span_b.t0}-{t1_b_str}" - ) - return True, "" - def _find_parallel_matches( self, patterns: list[str | AnyPattern | ParallelPattern], spans: list[Span] ) -> list[tuple[ParallelPattern, list[Span]]]: @@ -305,7 +267,6 @@ def _find_parallel_matches( """ matches = [] span_idx = 0 - span_names = [self._get_span_name(s.path) for s in spans] for pattern in patterns: if isinstance(pattern, ParallelPattern): @@ -346,7 +307,7 @@ async def __call__(self, ctx: ValidationContext) -> None: # Get all spans that are direct children of the target. Should already be sorted spans = ctx.trace.get_level(self.target) - span_names = [self._get_span_name(s.path) for s in spans] + span_names = [get_span_name(s.path) for s in spans] if not patterns: raise AssertionError(f"no patterns defined in seq! value: {self.value}") @@ -379,6 +340,8 @@ async def __call__(self, ctx: ValidationContext) -> None: # If matched, also validate that parallel patterns actually ran in parallel parallel_matches = self._find_parallel_matches(patterns, spans) for pattern, matched_spans in parallel_matches: - is_parallel, parallel_error = self._validate_parallel_spans(matched_spans, pattern.tolerance_ms) + is_parallel, parallel_error = validate_parallel_spans( + matched_spans, pattern.tolerance_ms, label=" in parallel!", + ) if not is_parallel: raise AssertionError(parallel_error) diff --git a/python/timbal/models.yaml b/python/timbal/models.yaml index b16a7467..f933317f 100644 --- a/python/timbal/models.yaml +++ b/python/timbal/models.yaml @@ -1,7 +1,7 @@ # Source of truth for all supported LLM models. # Pricing is in USD per 1M tokens (input / output). null = unknown or not publicly listed. # context_window is in tokens. null = unknown. -# Run scripts/generate_models.py to regenerate the Model Literal in llm_router.py. +# Run scripts/generate_models.py to regenerate the Model Literal in core/llm/router.py. # # Availability flags (optional; set one at most): # requires_activation: true — model exists but must be enabled in the provider console first diff --git a/python/timbal/platform/tool_proxy.py b/python/timbal/platform/tool_proxy.py index 2ffc293e..8d6b650b 100644 --- a/python/timbal/platform/tool_proxy.py +++ b/python/timbal/platform/tool_proxy.py @@ -32,7 +32,7 @@ def build_tool_proxy_headers() -> dict[str, str]: "x-timbal-version": __version__, } - # Match llm_router: always attach call_id when the context var is set (tool runs always set it). + # Match the llm router: always attach call_id when the context var is set (tool runs always set it). call_id = get_call_id() if call_id is not None: headers["x-timbal-call-id"] = call_id diff --git a/python/timbal/server/http.py b/python/timbal/server/http.py index 3af98b8a..7adfc61b 100644 --- a/python/timbal/server/http.py +++ b/python/timbal/server/http.py @@ -46,13 +46,18 @@ async def lifespan( app.state.single_session_guard = init_single_session_guard() # Voice warmup off the boot path: pre-import the voice stack (and pre-load - # the local turn-detection models when server-configured) so the first - # voice session doesn't pay those costs. No-op-ish for non-voice usage. + # the local turn-detection ONNX models) so the first voice session doesn't + # pay those costs. Gated on actual voice intent — non-voice deployments + # must not download/load ONNX models just because timbal[voice] is + # installed. The playground launcher opts its child servers in via + # TIMBAL_VOICE_WARMUP=1 (see voice.voice_warmup_intended). from ..core.agent import Agent - from .voice import warmup_voice_stack + from .voice import voice_warmup_intended, warmup_voice_stack warmup_task = ( - asyncio.create_task(warmup_voice_stack(app.state.voice_config)) if isinstance(runnable, Agent) else None + asyncio.create_task(warmup_voice_stack(app.state.voice_config)) + if isinstance(runnable, Agent) and voice_warmup_intended(runnable) + else None ) try: yield diff --git a/python/timbal/server/jobs.py b/python/timbal/server/jobs.py index 2b90a99f..b237413d 100644 --- a/python/timbal/server/jobs.py +++ b/python/timbal/server/jobs.py @@ -16,7 +16,7 @@ def __init__(self): self._jobs: dict[str, Job] = {} def create_job(self, runnable, params, job_id: str | None = None) -> tuple[str, Job]: - _job_id: str = job_id if job_id is not None else uuid7(as_type="str").replace("-", "") # type: ignore + _job_id: str = job_id if job_id is not None else uuid7(as_type="hex") # type: ignore queue = asyncio.Queue() task = asyncio.create_task(self._run(runnable, params, queue)) task.add_done_callback(lambda _: self._jobs.pop(_job_id, None)) diff --git a/python/timbal/server/playground.py b/python/timbal/server/playground.py index 6a2821fd..01279578 100644 --- a/python/timbal/server/playground.py +++ b/python/timbal/server/playground.py @@ -22,6 +22,7 @@ import argparse import json +import os import shutil import socket import subprocess @@ -109,9 +110,14 @@ def spawn(self, import_spec: str, port: int | None = None) -> dict: # cwd = the agent file's directory: `uv run` walks up from there to # find the agent project's pyproject/venv, and load_dotenv picks up # that project's .env — not the launcher's. + # TIMBAL_VOICE_WARMUP=1: playground children pre-load the voice + # stack so picking "Smart Turn" on first Start doesn't eat the + # ONNX/HuggingFace cold path; production servers gate warmup on + # actual voice intent (see server.voice.voice_warmup_intended). proc = subprocess.Popen( # noqa: S603 cmd, cwd=spec_path.parent, + env={**os.environ, "TIMBAL_VOICE_WARMUP": "1"}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, diff --git a/python/timbal/server/voice.py b/python/timbal/server/voice.py index 313bf22e..c32b28ac 100644 --- a/python/timbal/server/voice.py +++ b/python/timbal/server/voice.py @@ -227,6 +227,39 @@ def runnable_meta_for_voice_page(runnable: Any, import_spec: str) -> dict[str, A _VOICE_HTML_META_TOKEN = "__TIMBAL_VOICE_RUNNABLE_META_JSON__" +_TRUTHY = frozenset({"1", "true", "t", "yes", "y", "on"}) +_FALSY = frozenset({"0", "false", "f", "no", "n", "off"}) + + +def voice_warmup_intended(runnable: Any) -> bool: + """Whether server boot should pre-import the voice stack and pre-load ONNX models. + + Historically this ran for every Agent app whenever the ``timbal[voice]`` + extra was installed, which meant non-voice deployments (e.g. platform + images built from ``timbal[all]``) downloaded and loaded Smart Turn + + Namo + Silero at boot for nothing. + + Policy, in order: + + 1. ``TIMBAL_VOICE_WARMUP`` env: truthy forces warmup (the playground + launcher sets this for its child servers), falsy disables it. + 2. The runnable declares ``voice_config`` — clearly a voice app. + 3. Any ``TIMBAL_VOICE_*`` / ``ELEVENLABS_VOICE_ID`` env is set — the + deployment is voice-configured even if the runnable isn't. + 4. Otherwise: no warmup. + """ + override = os.environ.get("TIMBAL_VOICE_WARMUP", "").strip().lower() + if override in _TRUTHY: + return True + if override in _FALSY: + return False + if getattr(runnable, "voice_config", None) is not None: + return True + if os.environ.get("ELEVENLABS_VOICE_ID"): + return True + return any(k.startswith("TIMBAL_VOICE_") for k in os.environ) + + async def warmup_voice_stack(voice_config: VoiceConfig) -> None: """Background warmup at server boot so the first voice session starts fast. @@ -572,7 +605,7 @@ def build_voice_session( from .recording_upload import platform_recording_upload_hook on_saved = platform_recording_upload_hook() - session_id = uuid7(as_type="str").replace("-", "") + session_id = uuid7(as_type="hex") session_kwargs["session_id"] = session_id session_kwargs["recorder"] = CallRecorder( Path(recording_cfg.dir) / f"{session_id}.mp3", diff --git a/python/timbal/state/config.py b/python/timbal/state/config.py index 96253684..b4ed064a 100644 --- a/python/timbal/state/config.py +++ b/python/timbal/state/config.py @@ -1,6 +1,6 @@ from enum import StrEnum -from pydantic import BaseModel, SecretStr, model_validator +from pydantic import BaseModel, PrivateAttr, SecretStr, model_validator class PlatformAuthType(StrEnum): @@ -81,6 +81,11 @@ class PlatformConfig(BaseModel): Defaults to True when not explicitly configured. """ + _fully_resolved: bool = PrivateAttr(default=False) + """Set by resolve_platform_config() once env/file merging has completed, so + a config inherited across chained runs is not re-resolved (which would + re-read ~/.timbal files on every run).""" + @model_validator(mode="before") @classmethod def handle_aliases(cls, values): diff --git a/python/timbal/state/config_loader.py b/python/timbal/state/config_loader.py index 2ac375c9..a30c67a8 100644 --- a/python/timbal/state/config_loader.py +++ b/python/timbal/state/config_loader.py @@ -40,20 +40,37 @@ def _resolve_section_name(profile: str) -> str: return f"profile {profile}" +_file_config_cache: dict[str, FileConfig] = {} +"""Per-profile cache of ~/.timbal file reads. Only used when reading the real +config dir (config_dir is None) — explicit config_dir calls (tests) always +re-read. Cleared by force_refresh in resolve_platform_config.""" + + def load_file_config( profile: str | None = None, config_dir: Path | None = None, + force_refresh: bool = False, ) -> FileConfig: """Load configuration from ~/.timbal/config and ~/.timbal/credentials files. + Results for the real config dir are cached per profile: these files are read + on every RunContext creation (including once per run in chained sessions), + and disk I/O + configparser dominate the cost. Pass ``force_refresh=True`` + to re-read from disk and refresh the cache. + Args: profile: Profile name to load. If None, uses TIMBAL_PROFILE env var or falls back to "default". - config_dir: Override the config directory (for testing). + config_dir: Override the config directory (for testing). Disables caching. + force_refresh: Re-read files even when a cached result exists. """ if profile is None: profile = os.getenv("TIMBAL_PROFILE", "default") + cacheable = config_dir is None + if cacheable and not force_refresh and profile in _file_config_cache: + return _file_config_cache[profile] + base_dir = config_dir or TIMBAL_CONFIG_DIR config_path = base_dir / "config" credentials_path = base_dir / "credentials" @@ -110,12 +127,15 @@ def load_file_config( error=str(e), ) - return FileConfig( + result = FileConfig( base_url=base_url, api_key=api_key, org=org, sync_traces_enabled=sync_traces_enabled, ) + if cacheable: + _file_config_cache[profile] = result + return result def _merge_sync_traces_enabled(platform_config: PlatformConfig, file_config: FileConfig) -> None: @@ -181,7 +201,16 @@ def resolve_platform_config( if is_default_call and _default_config_resolved and not force_refresh: return _cached_default_config.model_copy() if _cached_default_config else None - file_config = load_file_config(profile=profile, config_dir=config_dir) + # A config that already went through this function (e.g. inherited from a + # previous run's context during session chaining) has every field filled — + # skip the file/env re-resolution entirely. + if platform_config is not None and platform_config._fully_resolved and not force_refresh: + return platform_config + + if force_refresh: + _file_config_cache.clear() + + file_config = load_file_config(profile=profile, config_dir=config_dir, force_refresh=force_refresh) logger.debug("Loaded file config.", file_config=file_config._asdict()) if not platform_config: @@ -235,6 +264,7 @@ def resolve_platform_config( if not org_id: logger.debug("No org_id found, skipping subject resolution.") _merge_sync_traces_enabled(platform_config, file_config) + platform_config._fully_resolved = True if is_default_call: _cached_default_config, _default_config_resolved = platform_config, True return platform_config @@ -271,6 +301,7 @@ def resolve_platform_config( _merge_sync_traces_enabled(platform_config, file_config) + platform_config._fully_resolved = True if is_default_call: _cached_default_config, _default_config_resolved = platform_config, True diff --git a/python/timbal/state/context.py b/python/timbal/state/context.py index 4bfff541..fc340daa 100644 --- a/python/timbal/state/context.py +++ b/python/timbal/state/context.py @@ -2,7 +2,7 @@ from pathlib import Path from typing import Any -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from uuid_extensions import uuid7 from ..errors import SpanNotFound @@ -53,7 +53,7 @@ class RunContext(BaseModel): ) id: str = Field( - default_factory=lambda: uuid7(as_type="str").replace("-", ""), # type: ignore + default_factory=lambda: uuid7(as_type="hex"), # type: ignore description="Unique identifier for the run.", ) parent_id: str | None = Field( @@ -89,19 +89,22 @@ def normalize_platform_config(cls, data: Any) -> Any: del data["timbal_platform_config"] return data - _base_path: Path | None = PrivateAttr(default=None) - _trace: Trace = PrivateAttr() - _tracing_provider: type[TracingProvider] = PrivateAttr() - _session_data: dict[str, Any] | None = PrivateAttr(default=None) - _resume_values: dict[str, Any] = PrivateAttr(default_factory=dict) - """Active resume values keyed by id, supplied via ``resume=`` to fulfill a - paused run. The id is an approval_id for an approval gate (value normalized - to ``ApprovalResolution``) or a suspension_id for a ``suspend()`` call (value - is arbitrary). The two id-spaces are disjoint hashes, so one store routes - unambiguously to the right consumer.""" - _used_resume_ids: set[str] = PrivateAttr(default_factory=set) - """Resume ids that matched a gate or ``suspend()`` during this run. Used to - warn about unrecognized resume values (typos, stale IDs) at run completion.""" + # NOTE — runtime state is stored as plain instance attributes assigned in + # model_post_init, NOT pydantic PrivateAttr declarations. PrivateAttr reads + # route through BaseModel.__getattr__ (~20x slower than __dict__ lookup) and + # _trace/_resume_values/_tracing_provider are read on every runnable call: + # _base_path: Path | None + # _trace: Trace + # _tracing_provider: type[TracingProvider] | None + # _session_data: dict | None + # _resume_values: dict — active resume values keyed by id, supplied via + # ``resume=`` to fulfill a paused run. The id is an approval_id for an + # approval gate (value normalized to ``ApprovalResolution``) or a + # suspension_id for a ``suspend()`` call (value is arbitrary). + # _used_resume_ids: set — resume ids that matched a gate or ``suspend()`` + # during this run; used to warn about unrecognized resume values. + # Do not add class-level annotations for them — pydantic would turn any + # annotated underscore name back into a (slow) private attribute. def pending_approvals(self) -> list[dict[str, Any]]: """Return metadata for every span currently waiting on approval. @@ -184,6 +187,11 @@ def model_post_init(self, __context: Any) -> None: """ from .config_loader import resolve_platform_config + # Plain instance attributes (see NOTE above). + self._base_path: Path | None = None + self._session_data: dict[str, Any] | None = None + self._resume_values: dict[str, Any] = {} + self._used_resume_ids: set[str] = set() self._trace = Trace() # Explicit provider set on the runnable — skip auto-detection entirely. @@ -256,9 +264,11 @@ async def _save_trace(self) -> None: tracing provider. Manual calls to this method may interfere with the framework's automatic tracing lifecycle. """ - # Sync session data to root span before saving + # Sync session data to root span before saving. Skip when the session + # was never populated ({} from get_session()) and never synced before — + # avoids a dump() per span completion for the common no-session case. root = self.root_span() - if root is not None and self._session_data is not None: + if root is not None and self._session_data is not None and (self._session_data or root.session is not None): from ..utils import dump root.session = self._session_data diff --git a/python/timbal/state/tracing/span.py b/python/timbal/state/tracing/span.py index 6ebb8724..4b9b2142 100644 --- a/python/timbal/state/tracing/span.py +++ b/python/timbal/state/tracing/span.py @@ -1,36 +1,107 @@ -from typing import Any +from typing import Any, ClassVar -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, computed_field +from ..._slots import SlotModel, dump_value -class Span(BaseModel): - model_config = ConfigDict( - arbitrary_types_allowed=True, - extra="allow", - ) +class Span(SlotModel): + """A single runnable invocation inside a run's trace. - path: str = Field( - ..., - description="The path of the runnable.", - ) - call_id: str = Field( - ..., - description="The call id of the runnable.", - ) - parent_call_id: str | None = Field( - None, - description="The parent call id of the runnable.", - ) - t0: int = Field( - ..., - description="The start time of the runnable.", - ) - t1: int | None = Field( - None, - description="The end time of the runnable. Will be None if the runnable has not yet completed.", + Plain ``__slots__`` class (constructed once per runnable call — hot path). + Preserves the old pydantic surface: + + - ``model_dump()`` with ``elapsed`` included and ``input``/``output``/ + ``memory``/``session`` replaced by their dumped versions when available. + - ``runnable``/``memory``/``session`` excluded from dumps (``memory`` and + ``session`` only appear via their ``_*_dump`` counterparts). + - Tolerant construction from serialized records: unknown keys are kept and + re-emitted on dump (old ``extra="allow"``), ``elapsed`` is recomputed. + - ``status`` is stored as-is: a live span holds a ``RunStatus``; a span + reloaded from a provider keeps the plain dict (intentional). + """ + + __slots__ = ( + "path", + "call_id", + "parent_call_id", + "t0", + "t1", + "input", + "status", + "output", + "error", + "usage", + "metadata", + "runnable", + "memory", + "session", + "_input_dump", + "_output_dump", + "_memory_dump", + "_prev_memory_dump", + "_session_dump", + "_extra", ) - @computed_field + _FIELDS = ("path", "call_id", "parent_call_id", "t0", "t1", "input", "status", "output", "error", "usage", "metadata") + + # pydantic-compat: evals introspect Span.model_fields.keys() to know which + # names address span properties (includes non-serialized fields). + model_fields: ClassVar[dict[str, None]] = dict.fromkeys(_FIELDS + ("runnable", "memory", "session")) + + def __init__( + self, + *, + path: str, + call_id: str, + t0: int, + parent_call_id: str | None = None, + t1: int | None = None, + input: Any = None, + status: Any = None, + output: Any = None, + error: Any = None, + usage: dict[str, int] | None = None, + metadata: dict[str, Any] | None = None, + runnable: Any = None, + memory: Any = None, + session: Any = None, + **extra: Any, + ) -> None: + # _extra must be set first: __getattr__ reads it for unknown-attribute + # lookups and would otherwise recurse while the slot is still unset. + extra.pop("elapsed", None) # Computed field in dumps; always recomputed from t0/t1. + object.__setattr__(self, "_extra", extra or None) + self.path = path + """The path of the runnable.""" + self.call_id = call_id + """The call id of the runnable.""" + self.parent_call_id = parent_call_id + """The parent call id of the runnable.""" + self.t0 = t0 + """The start time of the runnable.""" + self.t1 = t1 + """The end time of the runnable. Will be None if the runnable has not yet completed.""" + self.input = input + """The input of the runnable.""" + self.status = status + """The status of the runnable.""" + self.output = output + """The output of the runnable.""" + self.error = error + """The error of the runnable.""" + self.usage = usage if usage is not None else {} + """The usage of the runnable.""" + self.metadata = metadata if metadata is not None else {} + """Flexible metadata storage for run-specific metrics and data.""" + self.runnable = runnable + """A reference to the runnable being executed. Excluded from dumps. + Will be None when initializing traces from serialized data.""" + self.memory = memory + """Used by Agent to retrieve message histories between runs. Excluded from dumps.""" + # INTERNAL: accepted on construction for deserialization support. + # Do not access directly; use RunContext.get_session() instead. + self.session = session + @property def elapsed(self) -> int | None: """The elapsed time in milliseconds (t1 - t0). None if span is not yet completed.""" @@ -38,64 +109,40 @@ def elapsed(self) -> int | None: return None return self.t1 - self.t0 - input: Any = Field( - None, - description="The input of the runnable. Will be None if the runnable has not yet started or if there was an error gathering the input.", - ) - status: Any | None = Field( # Any to prevent circular import - None, - description="The status of the runnable.", - ) - output: Any = Field( - None, - description="The output of the runnable. Will be None if the runnable has not yet completed or if there was an error.", - ) - error: Any = Field( - None, - description="The error of the runnable. Will be None if the runnable has not yet completed or if there was no error.", - ) - usage: dict[str, int] = Field( - default_factory=dict, - description="The usage of the runnable.", - ) - metadata: dict[str, Any] = Field( - default_factory=dict, - description="Flexible metadata storage for run-specific metrics and data.", - ) - - runnable: Any = Field( - None, - description=( - "A reference to the runnable being executed. " - "Can be used to access runnable properties, background tasks, and other runtime attributes. " - "Will be None when initializing traces from serialized data." - ), - exclude=True, - ) - memory: Any = Field( - None, - description=( - "This field is used by Agent to retrieve message histories between runs. " - "It can also be used to overwrite what an llm can see and whatnot. " - ), - exclude=True, - ) - # INTERNAL: Exposed as a field instead of PrivateAttr for deserialization support. - # Do not access directly; use RunContext.get_session() instead. - session: Any = Field(None, exclude=True) + def __getattr__(self, name: str) -> Any: + # Only called when normal lookup fails: unset _*_dump slots and + # extra keys from deserialized records land here. + if not name.startswith("_"): + extra = object.__getattribute__(self, "_extra") + if extra and name in extra: + return extra[name] + raise AttributeError(f"{type(self).__name__!r} object has no attribute {name!r}") - _input_dump: Any = PrivateAttr() - """The dumped/serialized version of input for internal use.""" - _output_dump: Any = PrivateAttr() - """The dumped/serialized version of output for internal use.""" - _memory_dump: Any = PrivateAttr() - """The dumped/serialized version of memory for internal use.""" - _session_dump: Any = PrivateAttr() - """The dumped/serialized version of session for internal use.""" + def __setattr__(self, name: str, value: Any) -> None: + # Old pydantic config was extra="allow": handlers may stash arbitrary + # attributes on their span (read by siblings via step_span()). Unknown + # public names go to _extra and are included in model_dump(). + try: + object.__setattr__(self, name, value) + except AttributeError: + if name.startswith("_"): + raise + extra = object.__getattribute__(self, "_extra") + if extra is None: + extra = {} + object.__setattr__(self, "_extra", extra) + extra[name] = value - def model_dump(self, **kwargs) -> dict[str, Any]: - """Override model_dump to use dumped versions of input and output during serialization.""" - data = super().model_dump(**kwargs) # Pydantic ignores private attributes by default + def model_dump(self, mode: str = "python", **_kwargs: Any) -> dict[str, Any]: + """Serialize the span, preferring the dumped versions of input/output/memory/session.""" + data: dict[str, Any] = {} + for field in self._FIELDS: + data[field] = dump_value(getattr(self, field), mode) + if field == "t1": + data["elapsed"] = self.elapsed + extra = self._extra + if extra: + data.update(extra) # Use dumped versions if available, otherwise fall back to originals if hasattr(self, "_input_dump"): data["input"] = self._input_dump diff --git a/python/timbal/types/content/custom.py b/python/timbal/types/content/custom.py index fdbeed24..f25b4c3a 100644 --- a/python/timbal/types/content/custom.py +++ b/python/timbal/types/content/custom.py @@ -14,14 +14,23 @@ class CustomContent(BaseContent): type: Literal["custom"] = "custom" value: dict[str, Any] + # Anthropic server-side tool result blocks stored verbatim in memory (see + # AnthropicCollector.result()). Meaningless to other APIs — skip on + # cross-provider replay instead of sending an unknown block type. + _ANTHROPIC_SERVER_BLOCKS = ("web_search_tool_result",) + @override - def to_openai_responses_input(self, role: str, **kwargs: Any) -> dict[str, Any]: + def to_openai_responses_input(self, role: str, **kwargs: Any) -> dict[str, Any] | None: """See base class.""" + if self.value.get("type") in self._ANTHROPIC_SERVER_BLOCKS: + return None return self.value @override - def to_openai_chat_completions_input(self, **kwargs: Any) -> dict[str, Any]: + def to_openai_chat_completions_input(self, **kwargs: Any) -> dict[str, Any] | None: """See base class.""" + if self.value.get("type") in self._ANTHROPIC_SERVER_BLOCKS: + return None return self.value @override diff --git a/python/timbal/types/content/tool_use.py b/python/timbal/types/content/tool_use.py index cc171c65..855c36c5 100644 --- a/python/timbal/types/content/tool_use.py +++ b/python/timbal/types/content/tool_use.py @@ -1,5 +1,4 @@ import json -from ast import literal_eval from typing import Any, Literal # `override` was introduced in Python 3.12; use `typing_extensions` for compatibility with older versions @@ -29,11 +28,14 @@ def validate_input(cls, v: Any): return coerce_to_dict(v) @override - def to_openai_responses_input(self, **kwargs: Any) -> dict[str, Any]: + def to_openai_responses_input(self, **kwargs: Any) -> dict[str, Any] | None: """See base class.""" - # TODO Review is_server_tool_use + # Server-side tool calls are provider-internal execution records (e.g. + # Anthropic web search replayed after a fallback switch to OpenAI). + # They can't be represented in another provider's API; skip them — the + # assistant text already carries the results. if self.is_server_tool_use: - raise NotImplementedError("is_server_tool_use is not supported for OpenAI responses yet") + return None return { "call_id": self.id, "type": "function_call", @@ -42,10 +44,10 @@ def to_openai_responses_input(self, **kwargs: Any) -> dict[str, Any]: } @override - def to_openai_chat_completions_input(self, **kwargs: Any) -> dict[str, Any]: + def to_openai_chat_completions_input(self, **kwargs: Any) -> dict[str, Any] | None: """See base class.""" if self.is_server_tool_use: - raise ValueError("is_server_tool_use is not supported for OpenAI chat completions") + return None # see to_openai_responses_input data = {"id": self.id, "type": "function", "function": {"arguments": json.dumps(self.input), "name": self.name}} diff --git a/python/timbal/types/events/__init__.py b/python/timbal/types/events/__init__.py index de7d2b9e..15c163bc 100644 --- a/python/timbal/types/events/__init__.py +++ b/python/timbal/types/events/__init__.py @@ -1,7 +1,5 @@ # ruff: noqa: F401 -from typing import Annotated - -from pydantic import Field +from typing import Any from .approval import ApprovalEvent from .base import BaseEvent @@ -10,9 +8,30 @@ from .output import OutputEvent from .start import StartEvent -# Create a discriminated union of all possible event types. -# Pydantic will use the 'type' field to determine which model to use. -Event = Annotated[ - StartEvent | OutputEvent | DeltaEvent | ApprovalEvent | InteractionEvent, - Field(discriminator="type"), -] +# Union of all possible event types. Deserialization dispatches on the 'type' +# field via validate_event() (events are plain classes, not pydantic models). +Event = StartEvent | OutputEvent | DeltaEvent | ApprovalEvent | InteractionEvent + +_EVENT_TYPES: dict[str, type[BaseEvent]] = { + StartEvent.type: StartEvent, + OutputEvent.type: OutputEvent, + DeltaEvent.type: DeltaEvent, + ApprovalEvent.type: ApprovalEvent, + InteractionEvent.type: InteractionEvent, +} + + +def validate_event(data: BaseEvent | dict[str, Any]) -> Event: + """Rehydrate an event from its ``model_dump()`` wire form. + + Dispatches on the ``type`` discriminator. Nested structures (RunStatus, + DeltaItem) are rebuilt by the target class. Unknown keys are ignored, + matching the old pydantic ``extra="ignore"`` behavior. + """ + if isinstance(data, BaseEvent): + return data + event_type = data.get("type") + cls = _EVENT_TYPES.get(event_type) + if cls is None: + raise ValueError(f"Unknown event type {event_type!r}. Must be one of {sorted(_EVENT_TYPES)}.") + return cls(**{k: v for k, v in data.items() if k != "type"}) diff --git a/python/timbal/types/events/approval.py b/python/timbal/types/events/approval.py index 7459b787..72df2c51 100644 --- a/python/timbal/types/events/approval.py +++ b/python/timbal/types/events/approval.py @@ -1,6 +1,4 @@ -from typing import Any, Literal - -from pydantic import Field +from typing import Any from .base import BaseEvent @@ -8,39 +6,101 @@ class ApprovalEvent(BaseEvent): """Event emitted when runnable execution is waiting on human approval.""" - type: Literal["APPROVAL"] = "APPROVAL" + __slots__ = ( + "t0", + "approval_id", + "runnable_path", + "runnable_name", + "runnable_type", + "tool_call_id", + "input", + "input_schema", + "prompt", + "description", + "kind", + "ui", + "metadata", + ) + + type = "APPROVAL" + + _FIELDS = BaseEvent._FIELDS + ( + "t0", + "approval_id", + "runnable_path", + "runnable_name", + "runnable_type", + "tool_call_id", + "input", + "input_schema", + "prompt", + "description", + "kind", + "ui", + "metadata", + ) - t0: int - """Unix-ms timestamp at which approval was requested. Useful for SLA timers.""" - approval_id: str - """Stable identifier used to approve or deny this runnable invocation.""" - runnable_path: str - """Full runnable path that requires approval.""" - runnable_name: str - """Runnable name that requires approval.""" - runnable_type: str - """Runnable class/type that requires approval.""" - tool_call_id: str | None = None - """The LLM tool_call id that triggered this gate, when the approval happened - inside an agent tool. Lets the frontend correlate the approval card with the - exact tool_use block in the chat transcript. ``None`` for direct calls.""" - input: Any - """Validated (redacted, if configured) input that would be passed to the runnable. - The *values* for a structured approval card. Pair with ``input_schema`` to render - a typed form with zero per-tool frontend code.""" - input_schema: dict[str, Any] | None = None - """JSON Schema of the runnable's parameters (titles/descriptions/types). Render - ``input`` against this for a generic, typed approval form — Tier 0, no custom UI.""" - prompt: str | None = None - """Optional human-readable summary. Text fallback for CLIs/logs/non-rich clients.""" - description: str | None = None - """Optional runnable or policy description.""" - kind: str | None = None - """Renderer discriminator for a rich approval card (mirrors ``InteractionEvent.kind``). - The frontend dispatches ``(kind, ui)`` exactly like it does ``(kind, payload)`` for - interactions. ``None`` means render generically from ``input`` + ``input_schema``.""" - ui: dict[str, Any] | None = None - """Structured, presentation-only JSON for the card (title, fields, severity, ...). - Authored via the tool's ``approval_ui``. Already redacted; safe to render verbatim.""" - metadata: dict[str, Any] = Field(default_factory=dict) - """Additional policy metadata for future approval engines.""" + def __init__( + self, + *, + run_id: str, + path: str, + call_id: str, + t0: int, + approval_id: str, + runnable_path: str, + runnable_name: str, + runnable_type: str, + parent_run_id: str | None = None, + parent_call_id: str | None = None, + tool_call_id: str | None = None, + input: Any = None, + input_schema: dict[str, Any] | None = None, + prompt: str | None = None, + description: str | None = None, + kind: str | None = None, + ui: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + **_ignored: Any, + ) -> None: + super().__init__( + run_id=run_id, + path=path, + call_id=call_id, + parent_run_id=parent_run_id, + parent_call_id=parent_call_id, + ) + self.t0 = t0 + """Unix-ms timestamp at which approval was requested. Useful for SLA timers.""" + self.approval_id = approval_id + """Stable identifier used to approve or deny this runnable invocation.""" + self.runnable_path = runnable_path + """Full runnable path that requires approval.""" + self.runnable_name = runnable_name + """Runnable name that requires approval.""" + self.runnable_type = runnable_type + """Runnable class/type that requires approval.""" + self.tool_call_id = tool_call_id + """The LLM tool_call id that triggered this gate, when the approval happened + inside an agent tool. Lets the frontend correlate the approval card with the + exact tool_use block in the chat transcript. ``None`` for direct calls.""" + self.input = input + """Validated (redacted, if configured) input that would be passed to the runnable. + The *values* for a structured approval card. Pair with ``input_schema`` to render + a typed form with zero per-tool frontend code.""" + self.input_schema = input_schema + """JSON Schema of the runnable's parameters (titles/descriptions/types). Render + ``input`` against this for a generic, typed approval form — Tier 0, no custom UI.""" + self.prompt = prompt + """Optional human-readable summary. Text fallback for CLIs/logs/non-rich clients.""" + self.description = description + """Optional runnable or policy description.""" + self.kind = kind + """Renderer discriminator for a rich approval card (mirrors ``InteractionEvent.kind``). + The frontend dispatches ``(kind, ui)`` exactly like it does ``(kind, payload)`` for + interactions. ``None`` means render generically from ``input`` + ``input_schema``.""" + self.ui = ui + """Structured, presentation-only JSON for the card (title, fields, severity, ...). + Authored via the tool's ``approval_ui``. Already redacted; safe to render verbatim.""" + self.metadata = metadata if metadata is not None else {} + """Additional policy metadata for future approval engines.""" diff --git a/python/timbal/types/events/base.py b/python/timbal/types/events/base.py index 3ec5f914..a12634e2 100644 --- a/python/timbal/types/events/base.py +++ b/python/timbal/types/events/base.py @@ -1,20 +1,41 @@ -from pydantic import BaseModel, ConfigDict +from typing import Any +from ..._slots import SlotModel -class BaseEvent(BaseModel): - """Base class for all timbal events yielded during flow execution.""" - model_config = ConfigDict(extra="ignore") - type: str +class BaseEvent(SlotModel): + """Base class for all timbal events yielded during flow execution. + + Events are plain ``__slots__`` classes (not pydantic models) because they + are constructed on every runnable call and every streamed delta. Unknown + keyword arguments are ignored on construction (the old pydantic config was + ``extra="ignore"``) so wire payloads with extra keys keep deserializing. + """ + + __slots__ = ("run_id", "parent_run_id", "path", "call_id", "parent_call_id") + + type: str = "" """The type of the event. This will be very useful for serializing and deserializing events.""" - run_id: str - """The id of the run this event was emitted from.""" - parent_run_id: str | None = None - """The id of the parent run (if any).""" - path: str - """The path of the element that yielded this event.""" - call_id: str - """The id of the single execution in a run.""" - parent_call_id: str | None = None - """The id of the parent call if this event comes from a nested runnable.""" - \ No newline at end of file + + _FIELDS = ("type", "run_id", "parent_run_id", "path", "call_id", "parent_call_id") + + def __init__( + self, + *, + run_id: str, + path: str, + call_id: str, + parent_run_id: str | None = None, + parent_call_id: str | None = None, + **_ignored: Any, + ) -> None: + self.run_id = run_id + """The id of the run this event was emitted from.""" + self.parent_run_id = parent_run_id + """The id of the parent run (if any).""" + self.path = path + """The path of the element that yielded this event.""" + self.call_id = call_id + """The id of the single execution in a run.""" + self.parent_call_id = parent_call_id + """The id of the parent call if this event comes from a nested runnable.""" diff --git a/python/timbal/types/events/delta.py b/python/timbal/types/events/delta.py index ca972535..9216856e 100644 --- a/python/timbal/types/events/delta.py +++ b/python/timbal/types/events/delta.py @@ -6,65 +6,155 @@ providing better observability and control over streaming LLM responses. """ -from typing import Annotated, Any, Literal - -from pydantic import BaseModel, Field +from typing import Any +from ..._slots import SlotModel from .base import BaseEvent -class DeltaItem(BaseModel): - id: str - type: str +class DeltaItem(SlotModel): + __slots__ = ("id",) + + type: str = "" + + _FIELDS = ("id", "type") + + def __init__(self, *, id: str, **_ignored: Any) -> None: + self.id = id class ToolUse(DeltaItem): - type: Literal["tool_use"] = "tool_use" - name: str - input: str = "" - is_server_tool_use: bool = False + __slots__ = ("name", "input", "is_server_tool_use") + + type = "tool_use" + + _FIELDS = DeltaItem._FIELDS + ("name", "input", "is_server_tool_use") + + def __init__(self, *, id: str, name: str, input: str = "", is_server_tool_use: bool = False, **_ignored: Any) -> None: + super().__init__(id=id) + self.name = name + self.input = input + self.is_server_tool_use = is_server_tool_use class ToolUseDelta(DeltaItem): - type: Literal["tool_use_delta"] = "tool_use_delta" - input_delta: str + __slots__ = ("input_delta",) + + type = "tool_use_delta" + + _FIELDS = DeltaItem._FIELDS + ("input_delta",) + + def __init__(self, *, id: str, input_delta: str, **_ignored: Any) -> None: + super().__init__(id=id) + self.input_delta = input_delta class Text(DeltaItem): - type: Literal["text"] = "text" - text: str + __slots__ = ("text",) + + type = "text" + + _FIELDS = DeltaItem._FIELDS + ("text",) + + def __init__(self, *, id: str, text: str, **_ignored: Any) -> None: + super().__init__(id=id) + self.text = text class TextDelta(DeltaItem): - type: Literal["text_delta"] = "text_delta" - text_delta: str + __slots__ = ("text_delta",) + + type = "text_delta" + + _FIELDS = DeltaItem._FIELDS + ("text_delta",) + + def __init__(self, *, id: str, text_delta: str, **_ignored: Any) -> None: + super().__init__(id=id) + self.text_delta = text_delta class Thinking(DeltaItem): - type: Literal["thinking"] = "thinking" - thinking: str + __slots__ = ("thinking",) + + type = "thinking" + + _FIELDS = DeltaItem._FIELDS + ("thinking",) + + def __init__(self, *, id: str, thinking: str, **_ignored: Any) -> None: + super().__init__(id=id) + self.thinking = thinking class ThinkingDelta(DeltaItem): - type: Literal["thinking_delta"] = "thinking_delta" - thinking_delta: str + __slots__ = ("thinking_delta",) + + type = "thinking_delta" + + _FIELDS = DeltaItem._FIELDS + ("thinking_delta",) + + def __init__(self, *, id: str, thinking_delta: str, **_ignored: Any) -> None: + super().__init__(id=id) + self.thinking_delta = thinking_delta class Custom(DeltaItem): - type: Literal["custom"] = "custom" - data: Any + __slots__ = ("data",) + + type = "custom" + + _FIELDS = DeltaItem._FIELDS + ("data",) + + def __init__(self, *, id: str, data: Any = None, **_ignored: Any) -> None: + super().__init__(id=id) + self.data = data class ContentBlockStop(DeltaItem): - type: Literal["content_block_stop"] = "content_block_stop" + __slots__ = () + + type = "content_block_stop" + + +_DELTA_ITEM_TYPES: dict[str, type[DeltaItem]] = { + cls.type: cls + for cls in (ToolUse, ToolUseDelta, Text, TextDelta, Thinking, ThinkingDelta, Custom, ContentBlockStop) +} + + +def validate_delta_item(data: DeltaItem | dict[str, Any]) -> DeltaItem: + """Build a DeltaItem from a dict using the ``type`` discriminator.""" + if isinstance(data, DeltaItem): + return data + item_type = data.get("type") + cls = _DELTA_ITEM_TYPES.get(item_type) + if cls is None: + raise ValueError(f"Unknown delta item type {item_type!r}.") + return cls(**{k: v for k, v in data.items() if k != "type"}) class DeltaEvent(BaseEvent): - type: Literal["DELTA"] = "DELTA" - item: Annotated[ - ToolUse | ToolUseDelta | - Text | TextDelta | - Thinking | ThinkingDelta | - Custom | ContentBlockStop, - Field(discriminator="type") - ] + __slots__ = ("item",) + + type = "DELTA" + + _FIELDS = BaseEvent._FIELDS + ("item",) + + def __init__( + self, + *, + run_id: str, + path: str, + call_id: str, + item: DeltaItem | dict[str, Any], + parent_run_id: str | None = None, + parent_call_id: str | None = None, + **_ignored: Any, + ) -> None: + super().__init__( + run_id=run_id, + path=path, + call_id=call_id, + parent_run_id=parent_run_id, + parent_call_id=parent_call_id, + ) + self.item = validate_delta_item(item) diff --git a/python/timbal/types/events/interaction.py b/python/timbal/types/events/interaction.py index 54341a94..d09a6c52 100644 --- a/python/timbal/types/events/interaction.py +++ b/python/timbal/types/events/interaction.py @@ -1,6 +1,4 @@ -from typing import Any, Literal - -from pydantic import Field +from typing import Any from .base import BaseEvent @@ -15,27 +13,77 @@ class InteractionEvent(BaseEvent): ``resume={interaction_id: value}``. """ - type: Literal["INTERACTION"] = "INTERACTION" - - t0: int - """Unix-ms timestamp at which the run suspended. Useful for SLA timers.""" - interaction_id: str - """Stable identifier used to resume this suspension (the suspension_id).""" - kind: str - """Discriminator the frontend uses to pick a renderer (e.g. ``ask_user``).""" - runnable_path: str - """Full runnable path that suspended.""" - runnable_name: str - """Runnable name that suspended.""" - runnable_type: str - """Runnable class/type that suspended.""" - tool_call_id: str | None = None - """The LLM tool_call id that triggered this suspension, when it happened inside - an agent tool. Lets the frontend correlate the interaction with the exact - tool_use block in the chat transcript. ``None`` for direct (non-agent) calls.""" - payload: dict[str, Any] = Field(default_factory=dict) - """JSON-serializable data describing what the caller must supply.""" - response_schema: dict[str, Any] | None = None - """Optional JSON Schema describing the shape the resume value must match. - The frontend can validate the user's input client-side before resuming with - ``resume={interaction_id: value}``. ``None`` means any value is accepted.""" + __slots__ = ( + "t0", + "interaction_id", + "kind", + "runnable_path", + "runnable_name", + "runnable_type", + "tool_call_id", + "payload", + "response_schema", + ) + + type = "INTERACTION" + + _FIELDS = BaseEvent._FIELDS + ( + "t0", + "interaction_id", + "kind", + "runnable_path", + "runnable_name", + "runnable_type", + "tool_call_id", + "payload", + "response_schema", + ) + + def __init__( + self, + *, + run_id: str, + path: str, + call_id: str, + t0: int, + interaction_id: str, + kind: str, + runnable_path: str, + runnable_name: str, + runnable_type: str, + parent_run_id: str | None = None, + parent_call_id: str | None = None, + tool_call_id: str | None = None, + payload: dict[str, Any] | None = None, + response_schema: dict[str, Any] | None = None, + **_ignored: Any, + ) -> None: + super().__init__( + run_id=run_id, + path=path, + call_id=call_id, + parent_run_id=parent_run_id, + parent_call_id=parent_call_id, + ) + self.t0 = t0 + """Unix-ms timestamp at which the run suspended. Useful for SLA timers.""" + self.interaction_id = interaction_id + """Stable identifier used to resume this suspension (the suspension_id).""" + self.kind = kind + """Discriminator the frontend uses to pick a renderer (e.g. ``ask_user``).""" + self.runnable_path = runnable_path + """Full runnable path that suspended.""" + self.runnable_name = runnable_name + """Runnable name that suspended.""" + self.runnable_type = runnable_type + """Runnable class/type that suspended.""" + self.tool_call_id = tool_call_id + """The LLM tool_call id that triggered this suspension, when it happened inside + an agent tool. Lets the frontend correlate the interaction with the exact + tool_use block in the chat transcript. ``None`` for direct (non-agent) calls.""" + self.payload = payload if payload is not None else {} + """JSON-serializable data describing what the caller must supply.""" + self.response_schema = response_schema + """Optional JSON Schema describing the shape the resume value must match. + The frontend can validate the user's input client-side before resuming with + ``resume={interaction_id: value}``. ``None`` means any value is accepted.""" diff --git a/python/timbal/types/events/output.py b/python/timbal/types/events/output.py index b9113700..28b7d20c 100644 --- a/python/timbal/types/events/output.py +++ b/python/timbal/types/events/output.py @@ -1,6 +1,4 @@ -from typing import Any, Literal - -from pydantic import PrivateAttr +from typing import Any from ...types.run_status import RunStatus from .base import BaseEvent @@ -8,32 +6,66 @@ class OutputEvent(BaseEvent): """Event emitted when a step completes with its full output.""" - type: Literal["OUTPUT"] = "OUTPUT" - - input: Any - """The input arguments passed to the runnable.""" - status: RunStatus - """The status summary of the runnable after it completed.""" - output: Any - """The result of the runnable.""" - error: Any - """The error that occurred during the runnable.""" - t0: int - """The start time of the runnable in milliseconds.""" - t1: int - """The end time of the runnable in milliseconds.""" - usage: dict[str, int] - """The usage of the runnable.""" - metadata: dict[str, Any] - """Additional metadata about the runnable.""" - _input_dump: Any = PrivateAttr() - """The dumped/serialized version of input for internal use.""" - _output_dump: Any = PrivateAttr() - """The dumped/serialized version of output for internal use.""" - - def model_dump(self, **kwargs) -> dict[str, Any]: + + __slots__ = ("input", "status", "output", "error", "t0", "t1", "usage", "metadata", "_input_dump", "_output_dump") + + type = "OUTPUT" + + _FIELDS = BaseEvent._FIELDS + ("input", "status", "output", "error", "t0", "t1", "usage", "metadata") + + def __init__( + self, + *, + run_id: str, + path: str, + call_id: str, + status: RunStatus | dict[str, Any], + t0: int, + t1: int, + parent_run_id: str | None = None, + parent_call_id: str | None = None, + input: Any = None, + output: Any = None, + error: Any = None, + usage: dict[str, int] | None = None, + metadata: dict[str, Any] | None = None, + **_ignored: Any, + ) -> None: + super().__init__( + run_id=run_id, + path=path, + call_id=call_id, + parent_run_id=parent_run_id, + parent_call_id=parent_call_id, + ) + # Preserve the pydantic-era invariant that a status is required: several + # regression tests pin that an OutputEvent can never be built without one. + if isinstance(status, dict): + status = RunStatus(**status) + elif not isinstance(status, RunStatus): + raise ValueError(f"OutputEvent status is required and must be a RunStatus or dict, got {status!r}.") + self.input = input + """The input arguments passed to the runnable.""" + self.status = status + """The status summary of the runnable after it completed.""" + self.output = output + """The result of the runnable.""" + self.error = error + """The error that occurred during the runnable.""" + self.t0 = t0 + """The start time of the runnable in milliseconds.""" + self.t1 = t1 + """The end time of the runnable in milliseconds.""" + self.usage = usage if usage is not None else {} + """The usage of the runnable.""" + self.metadata = metadata if metadata is not None else {} + """Additional metadata about the runnable.""" + # _input_dump / _output_dump are assigned post-construction by the + # framework (dumped/serialized versions of input/output for internal use). + + def model_dump(self, mode: str = "python", **kwargs: Any) -> dict[str, Any]: """Override model_dump to use dumped versions of input and output during serialization.""" - data = super().model_dump(**kwargs) + data = super().model_dump(mode=mode, **kwargs) # Use dumped versions if available, otherwise fall back to originals if hasattr(self, "_input_dump"): data["input"] = self._input_dump diff --git a/python/timbal/types/events/start.py b/python/timbal/types/events/start.py index d151d2b8..ba24a439 100644 --- a/python/timbal/types/events/start.py +++ b/python/timbal/types/events/start.py @@ -1,9 +1,9 @@ -from typing import Literal - from .base import BaseEvent class StartEvent(BaseEvent): """Event emitted when a step starts execution.""" - type: Literal["START"] = "START" + __slots__ = () + + type = "START" diff --git a/python/timbal/types/message.py b/python/timbal/types/message.py index 43314141..c1ff7737 100644 --- a/python/timbal/types/message.py +++ b/python/timbal/types/message.py @@ -40,7 +40,7 @@ class Message: - OpenAI Responses: 'completed', 'max_output_tokens', 'content_filter', etc. """ - __slots__ = ("role", "content", "stop_reason") + __slots__ = ("role", "content", "stop_reason", "_cached_dump", "_cached_dump_len") def __init__(self, role: Any, content: Any, stop_reason: str | None = None) -> None: """Initialize a Message instance. @@ -53,6 +53,14 @@ def __init__(self, role: Any, content: Any, stop_reason: str | None = None) -> N object.__setattr__(self, "role", role) object.__setattr__(self, "content", content) object.__setattr__(self, "stop_reason", stop_reason) + # Serialized-form cache (see timbal.utils.serialization). Long + # conversations re-dump the same Message objects on every turn + # (span input dump, memory dump, LLM input dump); messages are + # immutable after construction except for in-place content appends + # (e.g. synthesized server tool results), so the cache is validated + # against len(content). + object.__setattr__(self, "_cached_dump", None) + object.__setattr__(self, "_cached_dump_len", -1) def __str__(self) -> str: if self.stop_reason: @@ -66,12 +74,14 @@ def to_openai_responses_input(self) -> list[dict[str, Any]]: inputs = [] message_content = [] for content_item in self.content: - if isinstance(content_item, ToolUseContent): - inputs.append(content_item.to_openai_responses_input()) - elif isinstance(content_item, ToolResultContent): - inputs.append(content_item.to_openai_responses_input()) + if isinstance(content_item, ToolUseContent | ToolResultContent): + item_input = content_item.to_openai_responses_input() + if item_input is not None: + inputs.append(item_input) else: - message_content.append(content_item.to_openai_responses_input(role=self.role)) + item_input = content_item.to_openai_responses_input(role=self.role) + if item_input is not None: + message_content.append(item_input) if message_content: # Role here should only be 'user' or 'assistant' inputs.append({"role": self.role, "content": message_content}) @@ -81,7 +91,7 @@ def to_openai_chat_completions_input( self, *, reasoning_as: Literal["omit", "reasoning_content"] = "omit", - ) -> dict[str, Any]: + ) -> dict[str, Any] | None: """Convert the message to OpenAI's chat completions api expected input format. Args: @@ -98,7 +108,9 @@ def to_openai_chat_completions_input( reasoning_parts: list[str] = [] for content_item in self.content: if isinstance(content_item, ToolUseContent): - tool_calls.append(content_item.to_openai_chat_completions_input()) + tool_call = content_item.to_openai_chat_completions_input() + if tool_call is not None: + tool_calls.append(tool_call) elif isinstance(content_item, ToolResultContent): return content_item.to_openai_chat_completions_input() elif isinstance(content_item, ThinkingContent): @@ -122,6 +134,11 @@ def to_openai_chat_completions_input( openai_input["tool_calls"] = tool_calls if reasoning_parts: openai_input["reasoning_content"] = "".join(reasoning_parts) + if len(openai_input) == 1: + # Every content item was skipped (e.g. server-side tool blocks on + # cross-provider replay, or thinking-only turns with omit). A bare + # role dict is invalid for the API — drop the turn entirely. + return None return openai_input def to_anthropic_input(self) -> dict[str, Any]: diff --git a/python/timbal/types/run_status.py b/python/timbal/types/run_status.py index bfe7f550..76e2bdfd 100644 --- a/python/timbal/types/run_status.py +++ b/python/timbal/types/run_status.py @@ -1,12 +1,21 @@ -from typing import Literal +from typing import Any -from pydantic import BaseModel +from .._slots import SlotModel +VALID_STATUS_CODES = frozenset({"success", "error", "cancelled", "timeout"}) -class RunStatus(BaseModel): - code: Literal["success", "error", "cancelled", "timeout",] - """The code associated with the run status.""" - reason: str | None = None - """The reason for the run status.""" - message: str | None = None - """The message associated with the run status.""" + +class RunStatus(SlotModel): + __slots__ = ("code", "reason", "message") + + _FIELDS = ("code", "reason", "message") + + def __init__(self, *, code: str, reason: str | None = None, message: str | None = None, **_ignored: Any) -> None: + if code not in VALID_STATUS_CODES: + raise ValueError(f"Invalid run status code {code!r}. Must be one of {sorted(VALID_STATUS_CODES)}.") + self.code = code + """The code associated with the run status.""" + self.reason = reason + """The reason for the run status.""" + self.message = message + """The message associated with the run status.""" diff --git a/python/timbal/utils/__init__.py b/python/timbal/utils/__init__.py index 7f37d2b6..dbe538c9 100644 --- a/python/timbal/utils/__init__.py +++ b/python/timbal/utils/__init__.py @@ -11,7 +11,7 @@ from .model import create_model_from_handler, issubclass_safe, resolve_default from .net import is_port_in_use from .schema import SupportedStringFormats, SupportedTypes, assert_never, is_list, transform_schema - from .serialization import coerce_to_dict, dump, safe_is_nan, sync_to_async_gen + from .serialization import coerce_to_dict, dump, invalidate_message_dump_caches, safe_is_nan, sync_to_async_gen __all__ = [ # import_spec @@ -28,6 +28,7 @@ "resolve_default", # serialization "dump", + "invalidate_message_dump_caches", "safe_is_nan", "coerce_to_dict", "sync_to_async_gen", @@ -50,6 +51,7 @@ "resolve_default": ".model", # serialization "dump": ".serialization", + "invalidate_message_dump_caches": ".serialization", "safe_is_nan": ".serialization", "coerce_to_dict": ".serialization", "sync_to_async_gen": ".serialization", diff --git a/python/timbal/utils/serialization.py b/python/timbal/utils/serialization.py index 21cdc2fc..e164975e 100644 --- a/python/timbal/utils/serialization.py +++ b/python/timbal/utils/serialization.py @@ -29,11 +29,13 @@ def safe_is_nan(value: Any) -> bool: _File = None _Message = None _FileContent = None +_Runnable = None def _ensure_types(): - global _File, _Message, _FileContent + global _File, _Message, _FileContent, _Runnable if _File is None: + from ..core.runnable import Runnable from ..types.content.file import FileContent from ..types.file import File from ..types.message import Message @@ -41,6 +43,7 @@ def _ensure_types(): _File = File _Message = Message _FileContent = FileContent + _Runnable = Runnable # Pre-allocated singleton to avoid creating exception objects on every File hit @@ -87,21 +90,37 @@ def _dump_sync(value: Any) -> Any: raise _NEEDS_ASYNC if isinstance(value, _Message): + # Per-message dump cache: long conversations re-dump the same Message + # objects several times per turn. Messages are immutable after + # construction except for in-place content appends, so validate the + # cache against len(content). The returned dict is a shared read-only + # snapshot — dump consumers never mutate the inner dicts. + content = value.content + if value._cached_dump is not None and value._cached_dump_len == len(content): + return value._cached_dump result = { "role": value.role, - "content": [_dump_sync(c) for c in value.content], + "content": [_dump_sync(c) for c in content], } if value.stop_reason is not None: result["stop_reason"] = value.stop_reason + object.__setattr__(value, "_cached_dump", result) + object.__setattr__(value, "_cached_dump_len", len(content)) return result - # Marker attribute check — O(1) vs O(n) MRO scan - if getattr(value, "_is_timbal_runnable", False): - return value.model_dump() - + # BaseModel branch comes before the marker probes: failed getattr on a + # pydantic model raises AttributeError inside pydantic's __getattr__, which + # is expensive when probing thousands of content items per dump. if isinstance(value, BaseModel): + if isinstance(value, _Runnable): + return value.model_dump() return {k: _dump_sync(v) for k, v in value.__dict__.items()} + # Slotted timbal models (events, Span, RunStatus) — plain classes with a + # class-attribute marker (cheap getattr); no __dict__; use model_dump() + if getattr(value, "__timbal_serializable__", False): + return _dump_sync(value.model_dump()) + if isinstance(value, tuple): return tuple(_dump_sync(v) for v in value) @@ -157,21 +176,32 @@ async def _dump_async(value: Any) -> Any: return result if isinstance(value, _Message): + # See _dump_sync: cached, len-validated snapshot. Also caches messages + # with File content after their (potentially expensive) persist step. + content = value.content + if value._cached_dump is not None and value._cached_dump_len == len(content): + return value._cached_dump result = { "role": value.role, - "content": await asyncio.gather(*[_dump_async(c) for c in value.content]), + "content": await asyncio.gather(*[_dump_async(c) for c in content]), } if value.stop_reason is not None: result["stop_reason"] = value.stop_reason + object.__setattr__(value, "_cached_dump", result) + object.__setattr__(value, "_cached_dump_len", len(content)) return result - if getattr(value, "_is_timbal_runnable", False): - return value.model_dump() - + # BaseModel before marker probes — see _dump_sync. if isinstance(value, BaseModel): + if isinstance(value, _Runnable): + return value.model_dump() items = await asyncio.gather(*[_dump_async(v) for v in value.__dict__.values()]) return dict(zip(value.__dict__.keys(), items, strict=False)) + # Slotted timbal models (events, Span, RunStatus) — no __dict__; use model_dump() + if getattr(value, "__timbal_serializable__", False): + return await _dump_async(value.model_dump()) + if isinstance(value, Path): return value.as_posix() @@ -188,6 +218,25 @@ async def _dump_async(value: Any) -> Any: # Public API # --------------------------------------------------------------------------- +def invalidate_message_dump_caches(value: Any) -> None: + """Reset the cached dump on any Message reachable in *value* (shallow containers). + + Hooks are allowed to mutate message content in place (e.g. a post_hook + rewriting the output text); the framework re-dumps afterwards and must not + serve the pre-mutation cache. Called after post_hook execution. + """ + _ensure_types() + if isinstance(value, _Message): + object.__setattr__(value, "_cached_dump", None) + object.__setattr__(value, "_cached_dump_len", -1) + elif isinstance(value, (list, tuple)): + for item in value: + invalidate_message_dump_caches(item) + elif isinstance(value, dict): + for item in value.values(): + invalidate_message_dump_caches(item) + + async def dump(value: Any) -> Any: """Dumps all models that live within a nested structure of arbitrary depth. @@ -200,6 +249,11 @@ async def dump(value: Any) -> Any: return await _dump_async(value) +_GEN_DONE = object() +"""Unique end-of-generator sentinel for sync_to_async_gen. Using None would +silently truncate sync generators that legitimately yield None.""" + + async def sync_to_async_gen( gen: Generator[Any, None, None], loop: asyncio.AbstractEventLoop, @@ -208,18 +262,22 @@ async def sync_to_async_gen( """Auxiliary function to convert a sync generator to an async generator. This function also shares the context of the caller to the executor. """ - while True: - # StopIteration is special in Python. It's used to implement generator protocol and can't - # be pickled/transferred across threads properly. By catching it explicitly in the executor - # function and converting it to a sentinel value, we avoid problematic exception propagation. - def _next(): - try: - return next(gen) - except StopIteration: - return None - value = await loop.run_in_executor(None, lambda: ctx.run(_next)) - if value is None: + # StopIteration is special in Python. It's used to implement generator protocol and can't + # be pickled/transferred across threads properly. By catching it explicitly in the executor + # function and converting it to a sentinel value, we avoid problematic exception propagation. + def _next(): + try: + return next(gen) + except StopIteration: + return _GEN_DONE + + def _next_with_ctx(): + return ctx.run(_next) + + while True: + value = await loop.run_in_executor(None, _next_with_ctx) + if value is _GEN_DONE: break yield value diff --git a/python/timbal/voice/session.py b/python/timbal/voice/session.py index dc2f006b..89c214d2 100644 --- a/python/timbal/voice/session.py +++ b/python/timbal/voice/session.py @@ -371,7 +371,7 @@ def __init__( self._output_audio_chunks: list[bytes] = [] # Persistent call recording (MP3 + manifest; see voice/recording.py). # Distinct from the in-memory record_audio seam above. - self.session_id = session_id or uuid7(as_type="str").replace("-", "") + self.session_id = session_id or uuid7(as_type="hex") self._recorder = recorder #: Wall-clock session start (set when run() begins); transcript offsets #: in the recording manifest and session_transcript are relative to it. @@ -605,7 +605,7 @@ def _start_llm_warmup(self) -> None: if not (isinstance(model, str) and "/" in model): return - from ..core.llm_router import warmup_llm_connection + from ..core.llm import warmup_llm_connection self._llm_warmup_task = asyncio.create_task(warmup_llm_connection(model)) diff --git a/scripts/audit_models.py b/scripts/audit_models.py index 622c4869..7ebff1b0 100644 --- a/scripts/audit_models.py +++ b/scripts/audit_models.py @@ -128,7 +128,7 @@ def _load_models() -> list[dict]: def _offline_audit(models: list[dict]) -> list[str]: """Return a list of structural error messages.""" errors: list[str] = [] - from timbal.core.llm_router import _PROVIDERS + from timbal.core.llm import _PROVIDERS seen: set[str] = set() for m in models: @@ -146,7 +146,7 @@ def _offline_audit(models: list[dict]) -> list[str]: errors.append(f"provider mismatch for {mid}: yaml={m.get('provider')} id={provider}") if provider not in _PROVIDERS: - errors.append(f"unknown provider '{provider}' for {mid} (not in llm_router._PROVIDERS)") + errors.append(f"unknown provider '{provider}' for {mid} (not in llm registry _PROVIDERS)") for cap in m.get("capabilities", []): if cap not in STANDARD_CAPABILITIES: @@ -295,7 +295,7 @@ async def _probe_completion( dedicated_only: bool, catalog: set[str] | None, ) -> tuple[Status, str]: - # Match llm_router: prefer max_completion_tokens on chat/completions. + # Match the llm router: prefer max_completion_tokens on chat/completions. r = await _probe_chat_completions( client, base_url, api_key, api_name, use_max_completion_tokens=True ) @@ -379,7 +379,7 @@ async def _audit_model( async def _live_audit(models: list[dict]) -> list[ModelResult]: - from timbal.core.llm_router import _PROVIDERS + from timbal.core.llm import _PROVIDERS by_provider: dict[str, list[dict]] = {} for m in models: diff --git a/uv.lock b/uv.lock index e2b0b7ed..b4318c05 100644 --- a/uv.lock +++ b/uv.lock @@ -18,52 +18,6 @@ supported-markers = [ "sys_platform == 'win32'", ] -[manifest] -members = [ - "ace", - "timbal", -] - -[[package]] -name = "ace" -source = { editable = "ace" } -dependencies = [ - { name = "anthropic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "cachetools", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "google-genai", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "openai", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "psycopg", extra = ["binary", "pool"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "timbal", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] - -[package.dev-dependencies] -dev = [ - { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pytest-asyncio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pytest-cov", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] - -[package.metadata] -requires-dist = [ - { name = "anthropic", specifier = ">=0.79.0" }, - { name = "cachetools", specifier = ">=7.0.4" }, - { name = "google-genai", specifier = ">=1.70.0" }, - { name = "httpx" }, - { name = "openai", specifier = ">=2.20.0" }, - { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.3" }, - { name = "pydantic-settings" }, - { name = "timbal", editable = "." }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "pytest", specifier = ">=9.0.2" }, - { name = "pytest-asyncio", specifier = ">=1.3.0" }, - { name = "pytest-cov", specifier = ">=7.0.0" }, -] - [[package]] name = "aioice" version = "0.10.2" @@ -182,15 +136,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/41/562a61d5a61fba3ffb273a115e249f1d8471b9515c59fcc38b4b9deda238/av-17.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b41647e42884bf543b8e8d0a1dabd4d1b006c99183eb1a2d7afc5b01f73eeff4", size = 21324700, upload-time = "2026-06-07T05:52:53.972Z" }, ] -[[package]] -name = "cachetools" -version = "7.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f4/8b/0d3945a13955303b81272f759a0331e54c5c793da455e6f5706b89d2639c/cachetools-7.1.4.tar.gz", hash = "sha256:437f55a4e0c1b01a4f3077cc470e6991d47430970e36fbcb77e2be0df4fc1cd6", size = 40085, upload-time = "2026-05-21T22:40:43.376Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8c/7b/1fc1c09cc0756cf25861a3be10565915953876da48bb228fb9a672b20a42/cachetools-7.1.4-py3-none-any.whl", hash = "sha256:323dc4127934744db5b54eb4924482d7edafbf9554e820d1531c2e08c0e4ef54", size = 16761, upload-time = "2026-05-21T22:40:41.845Z" }, -] - [[package]] name = "certifi" version = "2026.2.25" @@ -270,95 +215,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] -[[package]] -name = "charset-normalizer" -version = "3.4.7" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705, upload-time = "2026-04-02T09:26:02.191Z" }, - { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419, upload-time = "2026-04-02T09:26:03.583Z" }, - { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901, upload-time = "2026-04-02T09:26:04.738Z" }, - { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742, upload-time = "2026-04-02T09:26:06.36Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061, upload-time = "2026-04-02T09:26:08.347Z" }, - { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239, upload-time = "2026-04-02T09:26:09.823Z" }, - { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173, upload-time = "2026-04-02T09:26:10.953Z" }, - { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841, upload-time = "2026-04-02T09:26:12.142Z" }, - { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304, upload-time = "2026-04-02T09:26:13.711Z" }, - { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455, upload-time = "2026-04-02T09:26:14.941Z" }, - { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036, upload-time = "2026-04-02T09:26:16.478Z" }, - { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739, upload-time = "2026-04-02T09:26:17.751Z" }, - { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277, upload-time = "2026-04-02T09:26:18.981Z" }, - { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819, upload-time = "2026-04-02T09:26:20.295Z" }, - { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281, upload-time = "2026-04-02T09:26:21.74Z" }, - { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843, upload-time = "2026-04-02T09:26:22.901Z" }, - { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328, upload-time = "2026-04-02T09:26:24.331Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061, upload-time = "2026-04-02T09:26:25.568Z" }, - { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031, upload-time = "2026-04-02T09:26:26.865Z" }, - { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239, upload-time = "2026-04-02T09:26:28.044Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589, upload-time = "2026-04-02T09:26:29.239Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733, upload-time = "2026-04-02T09:26:30.5Z" }, - { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652, upload-time = "2026-04-02T09:26:31.709Z" }, - { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229, upload-time = "2026-04-02T09:26:33.282Z" }, - { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552, upload-time = "2026-04-02T09:26:34.845Z" }, - { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806, upload-time = "2026-04-02T09:26:36.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316, upload-time = "2026-04-02T09:26:37.672Z" }, - { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274, upload-time = "2026-04-02T09:26:38.93Z" }, - { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468, upload-time = "2026-04-02T09:26:40.17Z" }, - { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460, upload-time = "2026-04-02T09:26:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330, upload-time = "2026-04-02T09:26:42.554Z" }, - { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828, upload-time = "2026-04-02T09:26:44.075Z" }, - { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, - { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, - { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, - { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, - { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, - { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, - { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, - { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, - { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, - { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, - { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, - { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, - { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, - { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, - { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, - { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, - { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, - { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, - { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, - { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, - { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, - { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, - { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, - { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, - { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, - { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, - { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, - { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, - { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, - { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, - { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, - { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, - { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, - { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, - { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, - { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, - { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, - { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, - { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, - { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, - { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, - { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, - { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, -] - [[package]] name = "click" version = "8.4.2" @@ -646,24 +502,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/22/4222d7ddf3da30f363edaa98e329c2bce6c65497c9cb2810931c8b2c0fbc/fsspec-2026.6.0-py3-none-any.whl", hash = "sha256:02e0b71817df9b2169dc30a16832045764def1191b43dcff5bb85bdee212d2a1", size = 203949, upload-time = "2026-06-16T01:57:26.358Z" }, ] -[[package]] -name = "google-auth" -version = "2.53.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pyasn1-modules", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c6/ad/ff781329bbbdc0974a098d996e89c9e1f7024262f9e3eec442fbb9ad1ac6/google_auth-2.53.0.tar.gz", hash = "sha256:e7e6aa16f6bee7b2b264830fd04f08087a1d5a836df516251a5d15327b246c9c", size = 335844, upload-time = "2026-05-15T20:53:07.928Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/c9/db44165ba7c581268c6d46017ef63339110378305062830104fc7fa144cb/google_auth-2.53.0-py3-none-any.whl", hash = "sha256:6e7449917c599b35126a99ec268ec6880301f2fea41dce198fe8fd83ff642b68", size = 246071, upload-time = "2026-05-15T20:53:05.609Z" }, -] - -[package.optional-dependencies] -requests = [ - { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] - [[package]] name = "google-crc32c" version = "1.8.0" @@ -694,27 +532,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, ] -[[package]] -name = "google-genai" -version = "2.8.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "distro", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "google-auth", extra = ["requests"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "sniffio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "tenacity", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "websockets", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5b/52/0244e310812f3063d09d60b30ae29ab7df9343bd005744cd5eeaa6ba39b4/google_genai-2.8.0.tar.gz", hash = "sha256:37a9b3cb127d763e7f4ca47452ae3562c87728773bd1b149f7b559c239da2bc1", size = 564955, upload-time = "2026-06-03T22:55:38.397Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/de/747ad1aa49e902da9a4699081c282a1ed8ceed3b4d295fd99a6d286e09e4/google_genai-2.8.0-py3-none-any.whl", hash = "sha256:4da0a223a100f4b37f609a68b835e3326ab0fa313314dc0fd9d34e76ee293844", size = 832497, upload-time = "2026-06-03T22:55:36.598Z" }, -] - [[package]] name = "h11" version = "0.16.0" @@ -1391,90 +1208,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, ] -[[package]] -name = "psycopg" -version = "3.3.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "(python_full_version < '3.13' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform == 'win32')" }, - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, -] - -[package.optional-dependencies] -binary = [ - { name = "psycopg-binary", marker = "(implementation_name != 'pypy' and sys_platform == 'darwin') or (implementation_name != 'pypy' and sys_platform == 'linux') or (implementation_name != 'pypy' and sys_platform == 'win32')" }, -] -pool = [ - { name = "psycopg-pool", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] - -[[package]] -name = "psycopg-binary" -version = "3.3.4" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" }, - { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" }, - { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" }, - { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" }, - { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" }, - { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" }, - { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" }, - { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" }, - { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" }, - { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" }, - { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" }, - { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, - { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, - { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, - { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, - { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, - { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, - { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, - { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, - { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, - { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, - { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, - { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, - { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, - { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" }, - { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" }, - { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" }, - { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" }, - { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" }, - { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" }, - { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" }, - { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" }, - { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, - { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, - { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, - { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" }, - { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" }, - { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" }, - { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" }, - { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" }, - { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" }, - { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, -] - -[[package]] -name = "psycopg-pool" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/90/82/7a23d26039827ecd4ebe93905651029ddd307c5182ad59296dfb6f67b528/psycopg_pool-3.3.1.tar.gz", hash = "sha256:b10b10b7a175d5cc1592147dc5b7eec8a9e0834eb3ed2c4a92c858e2f51eb63c", size = 31661, upload-time = "2026-05-01T23:31:59.809Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/ed/89c2c620af0e1660354cd8aabf9f5b21f911597ce22acb37c805d6c86bc8/psycopg_pool-3.3.1-py3-none-any.whl", hash = "sha256:2af5b432941c4c9ad5c87b3fa410aec910ec8f7c122855897983a06c45f2e4b5", size = 40023, upload-time = "2026-05-01T23:31:53.136Z" }, -] - [[package]] name = "pyarrow" version = "23.0.1" @@ -1525,27 +1258,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/f2/c0e76a0b451ffdf0cf788932e182758eb7558953f4f27f1aff8e2518b653/pyarrow-23.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:527e8d899f14bd15b740cd5a54ad56b7f98044955373a17179d5956ddb93d9ce", size = 28365807, upload-time = "2026-02-16T10:14:03.892Z" }, ] -[[package]] -name = "pyasn1" -version = "0.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, -] - -[[package]] -name = "pyasn1-modules" -version = "0.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, -] - [[package]] name = "pycparser" version = "3.0" @@ -2085,21 +1797,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, ] -[[package]] -name = "requests" -version = "2.34.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "charset-normalizer", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "idna", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, -] - [[package]] name = "rich" version = "14.3.3" @@ -2332,15 +2029,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, ] -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - [[package]] name = "timbal" source = { editable = "." } @@ -2411,7 +2099,7 @@ dev = [ requires-dist = [ { name = "aiortc", marker = "extra == 'all'", specifier = ">=1.9.0" }, { name = "aiortc", marker = "extra == 'voice'", specifier = ">=1.9.0" }, - { name = "anthropic", specifier = ">=0.45.0" }, + { name = "anthropic", specifier = ">=0.83.0" }, { name = "fastapi", marker = "extra == 'all'", specifier = ">=0.115.8" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115.8" }, { name = "httpx", specifier = ">=0.28.0" }, @@ -2606,15 +2294,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] -[[package]] -name = "tzdata" -version = "2026.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, -] - [[package]] name = "urllib3" version = "2.6.3"