leaderboard.csv

Provider,Category,Pipeline,Reasoning,include_in_rankings,Overall,Short,Medium,Long,Cost_Per_Page,Grounding_Added_Cost_Per_Page,Confidence_Added_Cost_Per_Page,Confidence_AUGRC,Cost_Short,Cost_Medium,Cost_Long,P_Short,P_Medium,P_Long,R_Short,R_Medium,R_Long,Word_Grounding,Word_Grounding_Short,Word_Grounding_Medium,Word_Grounding_Long,Page_Grounding,Page_Grounding_Short,Page_Grounding_Medium,Page_Grounding_Long,Latency_S_Per_Doc_Short,Latency_S_Per_Doc_Medium,Latency_S_Per_Doc_Long
Claude Code (Opus 4.8 No-Evidence),Coding Agents,claude_code_extract_opus_4_8,high,false,87.09,90.08,79.21,88.07,0.1617,,,,0.2144,0.0823,0.0338,90.17,79.80,89.38,90.20,78.94,87.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.2,157.9,315.5
Claude Code (Opus 4.8),Coding Agents,claude_code_extract_opus_4_8_evidence,high,true,86.84,90.54,79.17,77.94,0.6324,0,,,0.8721,0.2802,0.0959,90.69,79.27,81.97,90.40,79.08,77.48,49.48,45.31,62.35,58.43,72.52,74.84,67.38,68.03,274.0,347.1,501.9
Claude Code (Opus 5.5),Coding Agents,claude_code_extract_opus_5_5_evidence,medium,true,86.06,89.63,77.96,80.83,0.1214,0,,,0.1659,0.0530,0.0192,89.62,77.77,80.60,89.69,78.34,81.06,71.55,73.57,65.54,74.87,76.95,79.27,71.36,76.47,73.6,95.6,227.9
Claude Code (Sonnet 5.5),Coding Agents,claude_code_extract_sonnet_5_5_evidence,medium,true,94.68,96.22,92.97,83.69,0.0569,0,,,0.0735,0.0237,0.0091,96.68,94.47,83.36,95.93,91.83,84.05,64.61,64.59,63.18,71.22,75.99,77.93,71.50,74.98,82.5,87.7,187.0
Codex (GPT-5.5 No-Evidence),Coding Agents,codex_code_extract_gpt_5_5_low,low,false,93.57,95.68,91.15,78.88,0.2783,,,,0.3579,0.1214,0.0434,96.01,93.64,94.03,95.51,90.16,78.58,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,70.3,82.6,122.0
Codex (GPT-5.5),Coding Agents,codex_code_extract_gpt_5_5_low_evidence,low,true,93.35,95.59,88.72,87.97,0.3363,0,,,0.4354,0.1406,0.0472,95.86,89.85,89.29,95.36,87.86,87.08,52.57,53.13,48.01,61.87,80.64,83.27,73.87,79.90,89.0,117.8,164.6
Codex (GPT-5.6 Luna),Coding Agents,codex_code_extract_gpt_5_6_luna_medium_evidence,medium,true,91.16,93.50,87.43,80.05,0.0120,0,,,0.0156,0.0048,0.0016,94.06,89.54,85.29,93.25,86.66,79.28,40.03,37.18,47.66,50.89,79.81,82.85,74.07,69.50,110.4,138.1,256.1
Codex (GPT-5.6 Sol),Coding Agents,codex_code_extract_gpt_5_6_sol_low_evidence,low,true,93.77,96.04,90.19,82.73,0.2165,0,,,0.2813,0.0870,0.0300,96.28,91.37,86.24,95.84,89.21,82.42,54.66,50.87,64.58,69.01,83.62,85.12,80.77,78.84,95.2,106.6,197.2
Codex (GPT-5.6 Terra),Coding Agents,codex_code_extract_gpt_5_6_terra_low_evidence,low,true,92.26,95.60,86.69,77.36,0.0945,0,,,0.1231,0.0376,0.0121,95.99,89.59,92.33,95.38,85.20,76.71,43.68,40.48,53.32,51.17,80.52,83.51,75.18,69.32,69.4,92.0,123.3
Codex (GPT-6 Luna),Coding Agents,codex_code_extract_gpt_6_luna_medium_evidence,medium,true,91.01,93.09,87.71,80.88,0.0097,0,,,0.0126,0.0040,0.0014,93.17,88.63,85.85,93.07,87.48,80.93,65.70,62.91,71.11,74.36,83.93,84.41,83.47,80.17,206.1,228.9,341.9
Codex (GPT-6 Sol),Coding Agents,codex_code_extract_gpt_6_sol_low_evidence,low,true,94.57,95.99,92.42,87.25,0.0980,0,,,0.1279,0.0388,0.0122,96.33,93.49,91.28,95.80,91.72,86.70,77.11,77.61,75.53,78.31,86.50,87.20,85.12,84.82,104.2,89.9,101.4
Codex (GPT-6.1 Sol),Coding Agents,codex_code_extract_gpt_6_1_sol_low_evidence,low,true,95.31,97.01,92.96,85.46,0.0713,0,,,0.0924,0.0287,0.0098,96.88,93.44,85.24,97.19,92.84,85.69,87.61,89.03,84.38,85.70,89.37,89.63,89.25,86.80,112.5,101.5,136.2
Codex (GPT-6.1 Terra),Coding Agents,codex_code_extract_gpt_6_1_terra_low_evidence,low,true,94.28,96.26,91.86,81.21,0.0468,0,,,0.0612,0.0179,0.0052,97.23,92.18,95.51,95.86,91.71,81.37,85.69,88.13,81.32,77.21,87.42,88.49,86.10,81.15,86.0,77.3,86.5
Datalab Balanced,Specialized APIs,datalab_extract_balanced,,true,85.70,89.40,85.08,42.04,0.0350,0,0,,0.0350,0.0350,0.0350,89.68,85.52,42.02,89.57,84.70,42.05,2.02,2.67,0.24,0.00,48.50,56.90,38.55,0.01,246.5,378.4,479.7
DeepSeek V4.1 Flash,Commercial VLM,deepseek_v4_1_flash_extract_oneshot_structured_output_file,off,true,87.11,94.44,81.49,22.23,0.0032,,,,0.0037,0.0017,0.0006,94.06,81.59,22.16,94.98,81.73,22.31,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,9.9,29.2,29.0
Extend Max Context + Review Agent,Specialized APIs,extend_extract_max,,true,94.49,96.16,93.95,76.02,0.0842,0,0.01,0.0101420407518998,0.0814,0.0929,0.0767,96.56,93.61,76.02,96.02,94.93,76.02,61.19,62.07,60.45,54.33,90.14,91.06,90.86,75.77,63.7,143.1,777.6
Extend Performance + Review Agent,Specialized APIs,extend_extract,,true,93.69,96.31,91.71,70.34,0.0544,0,0.01,0.0106878882625869,0.0514,0.0629,0.0500,96.63,91.25,75.13,96.26,92.70,70.24,61.48,61.89,61.84,55.18,89.59,90.95,90.10,70.90,64.3,144.2,464.1
Gemma4 12B,OSS,gemma4_12b_vllm_extract_oneshot_structured_output_file,off,true,45.39,53.62,30.82,13.16,,,,,,,,56.69,34.18,44.89,53.94,35.29,14.44,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,164.5,362.7,603.4
Gemma4 26B,OSS,gemma4_26b_vllm_extract_oneshot_structured_output_file,off,true,70.27,77.70,59.61,28.88,,,,,,,,78.44,65.89,50.10,77.29,57.71,25.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,601.2,767.9,746.7
Gemma4 E2B,OSS,gemma4_e2b_vllm_extract_oneshot_structured_output_file,off,true,51.85,63.71,28.96,14.58,,,,,,,,68.98,49.08,67.85,62.58,26.00,11.84,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,31.9,66.9,151.3
Gemma4 E4B,OSS,gemma4_e4b_vllm_extract_oneshot_structured_output_file,off,true,69.64,81.10,48.93,26.82,,,,,,,,82.30,62.50,71.15,80.72,46.61,25.45,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,29.7,80.1,162.6
GLM-4.6V Flash,OSS,glm_4_6v_flash_vllm_extract_oneshot_structured_output_file,on,true,45.19,47.28,44.52,22.17,,,,,,,,49.86,50.07,55.66,46.46,43.19,18.54,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,174.8,301.9,502.1
GLM-5.3 Flash,Commercial VLM,glm_5_3_flash_extract_oneshot_structured_output_file,max,true,80.75,96.30,51.56,27.83,0.0034,,,,0.0037,0.0021,0.0013,96.28,52.08,36.76,96.52,51.11,27.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,57.5,127.9,249.9
Google Gemini 3.5 Flash,Commercial VLM,gemini_3_5_flash_extract_oneshot_structured_output_file,low,true,79.84,87.87,69.76,27.90,0.0100,,,,0.0118,0.0069,0.0024,88.17,75.13,83.72,87.70,69.34,26.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,14.0,36.7,22.8
Google Gemini 3.8 Flash,Commercial VLM,gemini_3_8_flash_extract_oneshot_structured_output_file,low,true,80.71,88.14,72.23,28.70,0.0043,,,,0.0049,0.0031,0.0012,88.23,72.06,71.30,88.12,72.48,28.13,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,10.6,27.7,33.1
InternVL3.5 14B,OSS,internvl3_5_14b_vllm_extract_oneshot_structured_output_file,off,true,66.35,80.70,39.04,19.41,,,,,,,,83.06,67.40,70.23,79.83,33.60,17.08,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,
InternVL3.5 8B,OSS,internvl3_5_8b_vllm_extract_oneshot_structured_output_file,off,true,64.63,79.26,36.62,17.57,,,,,,,,82.37,65.17,61.77,78.05,31.70,15.16,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,
Kimi K3,Commercial VLM,kimi_k3_extract_oneshot_structured_output_file,max,true,83.17,94.64,69.64,4.99,0.0675,,,,0.0780,0.0294,0.0113,94.70,70.26,9.79,94.60,69.09,4.92,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,130.6,227.6,188.4
Kimi-VL A3B,OSS,kimi_vl_a3b_vllm_extract_oneshot_structured_output_file,off,true,57.77,69.86,34.61,18.94,,,,,,,,71.33,44.22,50.10,69.70,34.33,15.41,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,42.5,115.3,212.6
KYNEX (GPT-5.4),Coding Agents,kynex_gpt_5_4,off,false,92.97,94.29,92.48,78.77,0.0905,,,,0.1144,0.0356,0.0581,94.04,93.55,85.61,94.90,91.75,75.46,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,118.8,245.3,653.9
KYNEX (GPT-5.5),Coding Agents,kynex_gpt_5_5,medium,true,94.19,95.76,92.96,80.40,0.0880,,,,0.1115,0.0345,0.0568,95.38,93.82,89.05,96.27,92.46,78.77,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,110.5,235.8,628.4
KYNEX (GPT-6.1 Sol),Coding Agents,kynex_gpt_6_1_sol,medium,true,94.63,96.04,93.14,84.18,0.0935,,,,0.1180,0.0367,0.0600,95.84,94.19,90.57,96.42,92.97,81.30,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,125.3,258.7,695.2
Lift Datalab 9B,OSS,lift_extract,,true,78.43,87.64,65.69,24.78,,,,,,,,87.88,68.20,34.69,87.66,65.44,24.09,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,43.4,182.7,202.5
LlamaExtract Agentic,LlamaExtract,llamaextract_agentic,,true,95.65,96.06,94.79,94.73,0.0312,0,0,0.016342544027,0.0312,0.0312,0.0312,96.48,94.99,94.74,95.88,94.73,94.74,82.33,81.39,84.94,81.63,88.20,86.95,90.33,92.59,132.6,280.8,451.4
LlamaExtract Agentic Plus,LlamaExtract,llamaextract_agentic_plus,,true,96.17,97.02,94.30,94.65,0.0750,0,0,0.015038596008,0.0750,0.0750,0.0750,97.08,94.86,94.77,97.02,93.93,94.54,83.25,83.22,83.44,82.84,88.26,87.97,88.48,90.61,106.7,172.9,495.2
LlamaExtract Agentic Plus (Max Effort),LlamaExtract,llamaextract_agentic_plus_max_effort,,true,96.17,96.82,95.48,91.36,0.1911,0,0,,0.2401,0.0885,0.0765,96.93,95.68,91.37,96.82,95.41,91.36,81.78,80.58,85.55,78.74,90.02,89.92,90.67,88.10,67.7,127.2,575.2
LlamaExtract Cost-Effective,LlamaExtract,llamaextract_cost_effective,,true,94.21,95.49,91.58,90.86,0.0100,0,0,0.0233,0.0100,0.0100,0.0100,95.64,92.31,91.62,95.70,91.37,90.31,79.80,80.99,77.52,76.16,86.28,86.97,84.60,86.41,174.2,261.5,623.0
MiniCPM-V 4.5,OSS,minicpm_v_4_5_vllm_extract_oneshot_structured_output_file,off,true,45.85,57.39,25.30,1.13,,,,,,,,59.58,33.39,14.16,57.75,25.61,0.62,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,
NuExtract 2.0 8B,OSS,nuextract_2_8b_vllm_extract_oneshot_structured_output_file,,true,49.42,62.92,23.50,6.39,,,,,,,,68.28,60.16,75.93,61.03,21.09,4.34,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,,,
NuExtract3,OSS,nuextract3_extract,off,true,81.02,86.41,77.01,32.71,,,,,,,,86.20,82.31,80.62,87.39,74.89,29.04,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,47.8,91.0,190.7
OpenAI GPT-5.4 Nano,Commercial VLM,openai_gpt_5_4_nano_extract_oneshot_structured_output_file,off,true,74.90,77.43,76.37,35.81,0.0021,,,,0.0025,0.0012,0.0005,77.36,80.27,71.17,78.20,75.52,33.72,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,23.5,63.8,184.3
OpenAI GPT-6 Astra,Commercial VLM,openai_gpt_6_astra_reasoning_low_extract_oneshot_structured_output_file,low,true,91.91,97.22,90.56,31.70,0.1109,,,,0.1320,0.0677,0.0382,97.14,92.08,75.58,97.32,90.16,29.87,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,35.0,106.4,90.1
Parseur,Specialized APIs,parseur_extract_v3_vision,,true,88.51,92.76,78.83,82.45,0.0500,,,,0.0500,0.0500,0.0500,93.52,78.85,82.32,92.45,80.72,83.49,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,17.3,107.1,548.7
Pi (Tencent Hy4),Coding Agents,pi_extract_hy4_vision_evidence,high,true,80.66,82.97,74.43,82.00,0.1324,0,,,0.1793,0.0588,0.0224,82.97,74.44,82.00,83.00,74.57,82.00,79.53,79.67,76.60,92.19,76.67,77.81,72.67,82.24,687.8,591.5,698.6
Pulse (Effort),Specialized APIs,pulse_schema_effort,,true,96.07,96.83,94.70,93.24,0.1050,0,,,0.1050,0.1050,0.1050,96.85,94.93,93.71,96.86,94.60,92.96,14.94,12.58,20.80,15.96,83.62,81.32,87.95,90.08,118.0,249.7,507.3
Pulse (Non-effort),Specialized APIs,pulse_schema_non_effort,,true,94.51,95.39,92.49,93.31,0.0300,0,,,0.0300,0.0300,0.0300,95.71,93.34,93.91,95.25,91.86,92.86,13.37,11.56,18.03,13.47,81.44,80.87,82.29,84.08,174.7,244.9,499.3
Qwen3.5 0.8B,OSS,qwen3_5_0_8b_vllm_extract_oneshot_structured_output_file,off,true,46.63,57.44,26.98,6.65,,,,,,,,59.35,33.80,30.39,58.09,26.19,4.70,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,144.8,257.5,455.9
Qwen3.5 2B,OSS,qwen3_5_2b_vllm_extract_oneshot_structured_output_file,off,true,63.40,71.22,50.93,25.93,,,,,,,,72.16,57.50,55.62,72.04,50.58,23.56,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,141.2,307.9,675.3
Qwen3.5 35B,OSS,qwen3_5_35b_a3b_fp8_vllm_extract_oneshot_structured_output_file,off,true,88.00,93.52,85.27,31.73,,,,,,,,93.41,91.15,81.07,93.82,82.81,26.83,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,46.0,144.8,302.7
Qwen3.5 4B,OSS,qwen3_5_4b_vllm_extract_oneshot_structured_output_file,off,true,82.43,89.06,76.64,27.31,,,,,,,,88.98,85.70,84.33,89.58,73.28,24.57,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,65.0,163.4,272.6
Qwen3.5 9B,OSS,qwen3_5_9b_vllm_extract_oneshot_structured_output_file,off,true,86.10,92.31,81.12,32.22,,,,,,,,92.60,88.53,85.52,92.52,78.22,27.22,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,72.3,184.1,341.2
Qwen3.6 35B,OSS,qwen3_6_35b_a3b_fp8_vllm_extract_oneshot_structured_output_file,off,true,88.11,92.86,86.37,36.90,,,,,,,,93.12,93.13,90.57,92.89,83.38,31.73,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,63.2,132.2,348.3
Qwen3.8 27B,OSS,qwen3_8_27b_fp8_vllm_extract_oneshot_structured_output_file,off,true,89.75,94.68,87.54,38.45,,,,,,,,95.04,93.82,92.29,94.43,84.83,33.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,192.2,357.7,898.2
Qwen3.8 Flash Next,OSS,qwen3_8_flash_next_fp8_vllm_extract_oneshot_structured_output_file,off,true,89.88,94.82,87.81,37.74,,,,,,,,94.62,94.38,88.68,95.36,85.03,33.69,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,112.9,279.3,683.8
Reducto Deep Extract,Specialized APIs,reducto_deep_extract,,true,96.08,96.84,94.80,92.85,0.0635,0,0,0.024770593642,0.0672,0.0575,0.0469,96.74,95.17,92.28,96.99,94.45,93.45,50.76,53.49,46.08,40.19,69.75,72.92,62.27,68.96,134.1,182.3,511.1
Reducto Extract,Specialized APIs,reducto_extract,,true,83.79,92.35,73.29,27.28,0.0457,,0,,0.0476,0.0420,0.0385,92.83,74.24,55.68,92.18,73.38,26.96,0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,20.7,133.1,84.0
Reducto Extract (v4),Specialized APIs,reducto_extract_v4,,true,79.58,92.20,59.45,19.29,0.0459,0,0,,0.0476,0.0415,0.0376,92.40,61.37,41.58,92.17,59.52,18.72,54.97,60.53,49.35,16.30,75.28,87.47,57.38,18.59,34.7,127.4,96.9