Repository navigation
Expand file tree
/
Copy pathRESULTS.jsonl
More file actions
94 lines (94 loc) · 29.5 KB
/
Copy pathRESULTS.jsonl
File metadata and controls
94 lines (94 loc) · 29.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "RAW", "score": 18, "total": 20, "pass_rate": 0.9, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-08", "git_sha": "seed", "machine": "macbook-m-series", "note": "proof-of-method slice"}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "OURS", "score": 18, "total": 20, "pass_rate": 0.9, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-08", "git_sha": "seed", "machine": "macbook-m-series", "note": "zero tax vs RAW"}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "opencode", "score": 15, "total": 20, "pass_rate": 0.75, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-08", "git_sha": "seed", "machine": "macbook-m-series", "note": "opencode loop drops 3 tasks the model gets raw"}
{"benchmark": "humaneval-rs", "model": "Devstral-Small-24B", "arm": "RAW", "score": 5, "total": 5, "pass_rate": 1.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-08", "git_sha": "seed", "machine": "macbook-m-series", "note": null}
{"benchmark": "humaneval-rs", "model": "Devstral-Small-24B", "arm": "OURS", "score": 5, "total": 5, "pass_rate": 1.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-08", "git_sha": "seed", "machine": "macbook-m-series", "note": "zero tax after tool-surface fix"}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 21, "total": 40, "pass_rate": 0.525, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-08", "git_sha": "seed", "machine": "macbook-m-series", "note": "fixed opponent baseline"}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "RAW", "score": 3, "total": 8, "pass_rate": 0.375, "mean_output_tokens": 379, "excluded": false, "captured": "2026-07-10", "git_sha": "seed", "machine": "macbook-m-series", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "OURS", "score": 3, "total": 8, "pass_rate": 0.375, "mean_output_tokens": 379, "excluded": false, "captured": "2026-07-10", "git_sha": "seed", "machine": "macbook-m-series", "note": "zero tax on the hard board"}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-3B", "arm": "OURS", "score": 2, "total": 8, "pass_rate": 0.25, "mean_output_tokens": 412, "excluded": false, "captured": "2026-07-10", "git_sha": "seed", "machine": "macbook-m-series", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 2, "excluded": true, "captured": "2026-07-10", "git_sha": "seed", "machine": "macbook-m-series", "note": "EXCLUDED: degenerate lane (2 tok/task) under GPU contention \u2014 serving, not model"}
{"benchmark": "swe-bench-lite", "model": "Devstral-Small-24B", "arm": "OURS", "score": null, "total": 300, "pass_rate": null, "mean_output_tokens": null, "excluded": false, "captured": null, "git_sha": "pending", "machine": "macbook-m-series", "note": "harness wired (benchmarks/swe/run_ours.py), official swebench scorer \u2014 run pending"}
{"benchmark": "frontier-rs", "model": "Devstral-Small-24B", "arm": "OURS", "score": null, "total": 7, "pass_rate": null, "mean_output_tokens": null, "excluded": false, "captured": null, "git_sha": "pending", "machine": "macbook-m-series", "note": "gym committed + reference-verified \u2014 run pending"}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "RAW", "score": 43, "total": 50, "pass_rate": 0.86, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "OURS", "score": 46, "total": 50, "pass_rate": 0.92, "mean_output_tokens": 85, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "opencode", "score": 0, "total": 50, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": "shim-bypassed (raw endpoint, no toolcall_shim) \u2014 unfair, re-run pending"}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "RAW", "score": 21, "total": 40, "pass_rate": 0.525, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 15, "total": 40, "pass_rate": 0.375, "mean_output_tokens": 265, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "opencode", "score": 17, "total": 40, "pass_rate": 0.425, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": "shim-bypassed (raw endpoint, no toolcall_shim) \u2014 unfair, re-run pending"}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "hermes", "score": 9, "total": 40, "pass_rate": 0.225, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "aider", "score": 19, "total": 40, "pass_rate": 0.475, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "RAW", "score": 18, "total": 40, "pass_rate": 0.45, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "OURS", "score": 20, "total": 40, "pass_rate": 0.5, "mean_output_tokens": 82, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "opencode", "score": 1, "total": 40, "pass_rate": 0.025, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": "shim-bypassed (raw endpoint, no toolcall_shim) \u2014 unfair, re-run pending"}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "aider", "score": 20, "total": 40, "pass_rate": 0.5, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-3B", "arm": "RAW", "score": 13, "total": 40, "pass_rate": 0.325, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-3B", "arm": "OURS", "score": 29, "total": 40, "pass_rate": 0.725, "mean_output_tokens": 112, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-3B", "arm": "opencode", "score": 0, "total": 40, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": "shim-bypassed (raw endpoint, no toolcall_shim) \u2014 unfair, re-run pending"}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-3B", "arm": "aider", "score": 32, "total": 40, "pass_rate": 0.8, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "RAW", "score": 28, "total": 40, "pass_rate": 0.7, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "OURS", "score": 12, "total": 40, "pass_rate": 0.3, "mean_output_tokens": 161, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "opencode", "score": 22, "total": 40, "pass_rate": 0.55, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": "shim-bypassed (raw endpoint, no toolcall_shim) \u2014 unfair, re-run pending"}
{"benchmark": "humaneval-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "hermes", "score": 25, "total": 40, "pass_rate": 0.625, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "humaneval-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "aider", "score": 29, "total": 40, "pass_rate": 0.725, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-10", "git_sha": "01b48fbdc", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 563, "excluded": false, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 479, "excluded": false, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-3B", "arm": "OURS", "score": 2, "total": 8, "pass_rate": 0.25, "mean_output_tokens": 314, "excluded": false, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 6, "excluded": true, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "RAW", "score": 0, "total": 0, "pass_rate": null, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "OURS", "score": 5, "total": 8, "pass_rate": 0.625, "mean_output_tokens": 215, "excluded": false, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "opencode", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-11", "git_sha": "e9d2ec1c2", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "RAW", "score": 1, "total": 3, "pass_rate": 0.3333333333333333, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 187, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "opencode", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "hermes", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "aider", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "RAW", "score": 1, "total": 3, "pass_rate": 0.3333333333333333, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 187, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "opencode", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "hermes", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "aider", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "RAW", "score": 1, "total": 8, "pass_rate": 0.125, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 164, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "opencode", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "hermes", "score": 1, "total": 8, "pass_rate": 0.125, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "aider", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "RAW", "score": 3, "total": 8, "pass_rate": 0.375, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "OURS", "score": 3, "total": 8, "pass_rate": 0.375, "mean_output_tokens": 361, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "opencode", "score": 4, "total": 8, "pass_rate": 0.5, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "hermes", "score": 4, "total": 8, "pass_rate": 0.5, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "aider", "score": 3, "total": 8, "pass_rate": 0.375, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-14", "git_sha": "1859bc615", "machine": "Joels-MBP.lan", "note": null}
{"benchmark": "agent-solve-t1", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 6, "total": 10, "pass_rate": 0.6, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "9e5e556c4", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 2/2 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t2", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 2, "total": 10, "pass_rate": 0.2, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "9e5e556c4", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 2/2 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t1", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 8, "total": 15, "pass_rate": 0.5333333333333333, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "d89171cc6", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 3/3 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t2", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 0, "total": 15, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "d89171cc6", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 3/3 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t1", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 14, "total": 15, "pass_rate": 0.9333333333333333, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "1406e2ad7", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 3/3 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t2", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 1, "total": 15, "pass_rate": 0.06666666666666667, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "1406e2ad7", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 3/3 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t1", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 13, "total": 15, "pass_rate": 0.8666666666666667, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "0c20177ef", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 3/3 clean reps, memory ON, never stripped"}
{"benchmark": "agent-solve-t2", "model": "Qwen2.5-Coder-7B", "arm": "OURS", "score": 0, "total": 15, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "0c20177ef", "machine": "Joels-MacBook-Pro.local", "note": "whole-being agent/solve battery, 3/3 clean reps, memory ON, never stripped"}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "opencode", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-22", "git_sha": "27b28e71f", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 14, "total": 40, "pass_rate": 0.35, "mean_output_tokens": 269, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "OURS", "score": 19, "total": 40, "pass_rate": 0.475, "mean_output_tokens": 81, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-3B", "arm": "OURS", "score": 27, "total": 40, "pass_rate": 0.675, "mean_output_tokens": 126, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "OURS", "score": 25, "total": 40, "pass_rate": 0.625, "mean_output_tokens": 119, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "RAW", "score": 0, "total": 0, "pass_rate": null, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Qwen2.5-Coder-14B", "arm": "OURS", "score": 37, "total": 40, "pass_rate": 0.925, "mean_output_tokens": 91, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Devstral-Small-24B", "arm": "RAW", "score": 0, "total": 1, "pass_rate": 0.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "humaneval-rs", "model": "Devstral-Small-24B", "arm": "OURS", "score": 35, "total": 40, "pass_rate": 0.875, "mean_output_tokens": 93, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Hermes-3-Llama-3.1-8B", "arm": "OURS", "score": 1, "total": 8, "pass_rate": 0.125, "mean_output_tokens": 419, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-1.5B", "arm": "OURS", "score": 0, "total": 8, "pass_rate": 0.0, "mean_output_tokens": 324, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-3B", "arm": "OURS", "score": 2, "total": 8, "pass_rate": 0.25, "mean_output_tokens": 389, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "qwen3.5-4b-code-forged (OURS-forged)", "arm": "OURS", "score": 2, "total": 8, "pass_rate": 0.25, "mean_output_tokens": 585, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "RAW", "score": 0, "total": 0, "pass_rate": null, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Qwen2.5-Coder-14B", "arm": "OURS", "score": 4, "total": 8, "pass_rate": 0.5, "mean_output_tokens": 291, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "RAW", "score": 0, "total": 0, "pass_rate": null, "mean_output_tokens": null, "excluded": true, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "hard-rs", "model": "Devstral-Small-24B", "arm": "OURS", "score": 3, "total": 8, "pass_rate": 0.375, "mean_output_tokens": 443, "excluded": false, "captured": "2026-07-23", "git_sha": "5d9adfde0", "machine": "Joels-MacBook-Pro.local", "note": null}
{"benchmark": "swe-bench-lite", "model": "unsloth/Devstral-Small-2507-GGUF", "arm": "OURS-agent", "score": 0, "total": 1, "pass_rate": 0.0, "excluded": false, "captured": "2026-07-23", "git_sha": "69c63890f", "machine": "Joels-MacBook-Pro", "note": "flask-4045 UNRESOLVED (official harness, image-local run). Near-miss anatomy: found the EXACT gold fix (ValueError on dotted Blueprint name, correct site) but smeared it — 2 duplicate insertions incl. a broken trailing __init__ redefinition that breaks the class. Defect class: append-style edits instead of precise replacement + never ran the repo's test suite before finishing. Both are verify-correct gene targets (training queued tonight)."}
{"benchmark": "swe-bench-lite", "model": "unsloth/Devstral-Small-2507-GGUF", "arm": "OURS-team", "score": 0, "total": 1, "pass_rate": 0.0, "excluded": false, "captured": "2026-07-23", "git_sha": "51ffa2620", "machine": "Joels-MacBook-Pro", "note": "flask-4045 team arm UNRESOLVED but a DIFFERENT, better miss than solo: Anwen's re-solve was CLEAN (19 acts vs 40; single correct-site insertion, 488B vs 1959B smear — first run after learn-mode admitted her run-1 experience). Asha's review (40 acts) preserved it. Remaining gap: fix covers dotted NAME only; gold also validates dotted ENDPOINTS (add_url_rule) — the FAIL_TO_PASS endpoint test still fails. Also 111 spurious werkzeug-deprecation errors polluted her test-feedback signal."}
{"benchmark": "hard-rs", "model": "ornith-ai/Ornith-1.5-35B-A3B-GGUF", "arm": "OURS-citizen", "score": 6, "total": 8, "pass_rate": 0.75, "excluded": false, "captured": "2026-08-23", "git_sha": "d4d26a717", "machine": "Joels-MacBook-Pro-M5-64GB", "note": "Take 3 — first honest hard-rs number (takes 1-2 infra-poisoned by #2382/#2384 grading defects, discarded unrecorded). Citizen live cognition via cognition/eval, #312 ephemeral CoW root, glass-boxed both directions (fail hand-verified genuine, pass = defect-2's witness task). Contention caveat: Atlas's own SWE round ran concurrently (reboot wiped sleep mode) — 45.7% mean cache hit, 47 tok/s mean decode. run 0fd92d32."}
{"benchmark": "terminal-bench-2.1", "model": "Ornith-1.5-35B-A3B", "arm": "OURS", "score": null, "total": 7, "pass_rate": null, "mean_output_tokens": null, "excluded": false, "captured": "2026-08-24", "git_sha": "c981a0879", "machine": "macbook-m5-64gb", "note": "gold-gated 7/53 subset (46 env-blocked tasks named in tb-goldgate.csv); round d9d9c555 IN FLIGHT — one-command round, learn mode, lived sampling"}
{"benchmark": "terminal-bench-2.1", "model": "Ornith-1.5-35B-A3B", "arm": "mini-swe-stock", "score": null, "total": 7, "pass_rate": null, "mean_output_tokens": null, "excluded": false, "captured": "2026-08-24", "git_sha": "c981a0879", "machine": "macbook-m5-64gb", "note": "QUEUED after OURS: mini-swe-agent 2.4.6 + unmodified ggml-org llama-server b10612, default flags (-c 65536 documented exception) — their whole world"}
{"benchmark": "terminal-bench-2.1", "model": "Ornith-1.5-35B-A3B", "arm": "OURS", "score": 5, "total": 7, "pass_rate": 0.714, "mean_output_tokens": null, "excluded": false, "captured": "2026-08-24", "git_sha": "e06241469", "machine": "macbook-m5-64gb", "note": "GOLD-GATED subset (7 of 53 tasks provably gradeable on this host; 46 env-blocked tasks named in receipts — NOT a 53-task claim). Run 706215f4: 5 passes incl. five consecutive after the mid-round working-memory fix (act counts 32→16→18→11→5→11); 2 misses: cancel-async-tasks = harness env-fail (bare python → exit 127; interpreter projection fixed, retake owed), llm-inference-batching-scheduler = budget exhaustion on the pre-fix starved mind (retake owed). Round interrupted+resumed 3x with all grades kept (one-command round). Replication: continuum benchmark/round --benchmark tb21-gold7.jsonl --persona <id>"}
{"benchmark": "terminal-bench-2.1", "model": "Ornith-1.5-35B-A3B", "arm": "mini-swe-stock", "score": 6, "total": 7, "pass_rate": 0.857, "mean_output_tokens": null, "excluded": false, "captured": "2026-08-24", "git_sha": "e06241469", "machine": "macbook-m5-64gb", "note": "THEIR WHOLE WORLD: mini-swe-agent 2.4.6 + unmodified ggml-org llama-server b10612 default flags (-c 65536 documented exception), same 7 gold-gated tasks + oracles, ~5 min/task. HONEST READING: on the 6 tasks where both arms had fair envs it is DEAD EVEN (5 passes each, both missed llm-inference-batching-scheduler); the 1-task gap is cancel-async-tasks, which this arm sat on the interpreter-FIXED harness while OURS was graded on the broken one — OURS retake pending. Driver receipts: 3 harness-env bugs on the opponent's side were fixed before any row minted (wizard, cost-tracker, exit prompt) — same env-fail protection both directions."}
{"benchmark": "terminal-bench-2.1", "model": "Ornith-1.5-35B-A3B", "arm": "OURS", "score": 7, "total": 7, "pass_rate": 1.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-08-25", "git_sha": "e06241469", "machine": "macbook-m5-64gb", "note": "ROUND 2 — THE RETAKE CURVE (memory claim, disclosed: same 7 gold-gated tasks as round 1's 71%; fresh-task transfer curve is the separate obligation as env fixes unlock the 46 blocked tasks). One night of lessons between rounds, no code changes to her mind mid-round. Act counts round1→round2: cancel-async env-fail→PASS(21), code-from-image 16→2, db-wal 18→23, batching-scheduler MISS→PASS(20) — the task BOTH arms (incl. mini-swe-stock) missed fresh, cracked via her own lesson carrying the oracle's FileNotFoundError evidence. Beats mini-swe-stock's 86% first-sight (their harness cannot retake — frozen scaffolds have a score, ours has a slope). Run d4bfc787; replication: continuum benchmark/round --benchmark tb21-gold7.jsonl --persona <id> --fresh true"}
{"benchmark": "terminal-bench-2.1", "model": "Ornith-1.5-35B-A3B", "arm": "OURS", "score": 9, "total": 9, "pass_rate": 1.0, "mean_output_tokens": null, "excluded": false, "captured": "2026-08-25", "git_sha": "e06241469", "machine": "macbook-m5-64gb", "note": "CUMULATIVE gold-gated coverage after round 2 + TRANSFER round: 9/9 distinct tasks passed at latest attempt (7 retake + 2 NEVER-SEEN: constraints-scheduling 7 acts first-sight, extract-moves-from-video 1 act first-sight — the fresh-task/transfer curve's opening points; single-act video pass flagged for post-mortem verification of oracle strictness per standing rhythm). Retake curve and transfer curve now BOTH nonzero — memory AND generalization measured separately. Runs 706215f4+d4bfc787+08b95e84."}