{ "source": "https://dach.peerbench.ai", "generated_at": "2026-07-23T14:09:15.229Z", "last_run": "2026-07-23", "benchmarks": [ { "key": "germeval_de", "display": "GermEval NER (DE)", "description": "Native German named-entity recognition (person/location/org/misc) emitted as JSON; seqeval micro-F1 excluding MISC." }, { "key": "include_de", "display": "INCLUDE (DE)", "description": "CohereLabs include-base-44 German split, 4-option MCQ, generate_until variant." }, { "key": "mmlu_prox_de", "display": "MMLU-ProX (DE)", "description": "German MMLU-Pro, 10-option MCQ across subjects." }, { "key": "mmmlu_de_gen", "display": "MMMLU (DE)", "description": "German MMLU, 4-option MCQ, generate_until variant." }, { "key": "musr_de", "display": "MuSR (DE)", "description": "German MuSR multistep soft reasoning (murder_mystery/object_placements/team_allocation), frozen v1 translation, generate_until cot+; primary metric accuracy." }, { "key": "sb10k_de", "display": "SB10K (DE)", "description": "Native German Twitter sentiment (positive/neutral/negative), generate_until variant; primary metric MCC." }, { "key": "scala_de", "display": "ScaLA (DE)", "description": "Native German linguistic acceptability — is the sentence grammatical (ja/nein)? generate_until; accuracy headline (MCC recorded)." } ], "models": [ { "slug": "google/gemini-3.1-pro-preview", "display": "Gemini 3.1 Pro", "vendor": "Google" }, { "slug": "openai/gpt-5.5", "display": "GPT-5.5", "vendor": "OpenAI" }, { "slug": "anthropic/claude-opus-4.8", "display": "Opus 4.8", "vendor": "Anthropic" }, { "slug": "google/gemini-3-flash-preview", "display": "Gemini 3 Flash Preview", "vendor": "Google" }, { "slug": "google/gemini-3.6-flash", "display": "Gemini 3.6 Flash", "vendor": "Google" }, { "slug": "google/gemini-3.5-flash", "display": "Gemini 3.5 Flash", "vendor": "Google" }, { "slug": "google/gemini-3.1-flash-lite", "display": "Gemini 3.1 Flash-Lite", "vendor": "Google" }, { "slug": "google/gemma-4-31b-it", "display": "Gemma 4 31B", "vendor": "Google" }, { "slug": "google/gemini-3.5-flash-lite", "display": "Gemini 3.5 Flash-Lite", "vendor": "Google" }, { "slug": "qwen/qwen3.6-35b-a3b", "display": "Qwen3.6 35B-A3B", "vendor": "Alibaba" }, { "slug": "deepseek/deepseek-v4-flash", "display": "DeepSeek V4 Flash", "vendor": "DeepSeek" }, { "slug": "google/gemini-2.5-flash", "display": "Gemini 2.5 Flash", "vendor": "Google" }, { "slug": "deepseek/deepseek-v4-pro", "display": "DeepSeek V4 Pro", "vendor": "DeepSeek" }, { "slug": "google/gemma-4-26b-a4b-it", "display": "Gemma 4 26B A4B", "vendor": "Google" }, { "slug": "tencent/hy3-preview", "display": "Tencent HY3-Preview", "vendor": "Tencent" }, { "slug": "anthropic/claude-haiku-4.5", "display": "Claude Haiku 4.5", "vendor": "Anthropic" }, { "slug": "google/gemini-2.5-flash-lite", "display": "Gemini 2.5 Flash-Lite", "vendor": "Google" }, { "slug": "google/gemma-4-12b-it", "display": "Gemma 4 12B", "vendor": "Google" }, { "slug": "qwen/qwen3.5-9b", "display": "Qwen3.5 9B", "vendor": "Alibaba" }, { "slug": "qwen/qwen3-14b", "display": "Qwen3 14B", "vendor": "Alibaba" }, { "slug": "minimax/minimax-m2.7", "display": "MiniMax M2.7", "vendor": "MiniMax" }, { "slug": "anthropic/claude-fable-5", "display": "Fable 5", "vendor": "Anthropic" }, { "slug": "anthropic/claude-sonnet-5", "display": "Sonnet 5", "vendor": "Anthropic" }, { "slug": "qwen/qwen3.7-max", "display": "Qwen3.7 Max", "vendor": "Alibaba" }, { "slug": "moonshotai/kimi-k2.6", "display": "Kimi K2.6", "vendor": "Moonshot" }, { "slug": "xiaomi/mimo-v2.5-pro", "display": "MiMo V2.5 Pro", "vendor": "Xiaomi" }, { "slug": "z-ai/glm-5.1", "display": "GLM-5.1", "vendor": "Z.ai" }, { "slug": "openai/gpt-oss-120b", "display": "gpt-oss-120b", "vendor": "OpenAI" }, { "slug": "x-ai/grok-4.3", "display": "grok-4.3", "vendor": "Unknown" }, { "slug": "mistralai/ministral-14b-2512", "display": "Ministral 14B", "vendor": "Mistral" }, { "slug": "google/gemma-3-12b-it", "display": "gemma-3-12b-it", "vendor": "Unknown" } ], "rows": [ { "kind": "headline", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.927147130038456, "items_effective": 14042, "items_total": 14042, "reasoning_off": false, "median_latency_ms": 4849.3, "avg_prompt_tokens": 172.5, "avg_response_tokens": 158.5, "avg_reasoning_tokens": 267.6, "cost_usd": 76.027902, "quantization": "provider-internal", "run_date": "2026-07-13", "run_id": "2026-07-13T09-50-23__vertex_ai__gemini-3.1-pro-preview__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.9210938612733228, "items_effective": 14042, "items_total": 14042, "reasoning_off": false, "median_latency_ms": 4530.4, "avg_prompt_tokens": 198.9, "avg_response_tokens": 74.9, "avg_reasoning_tokens": 91.9, "cost_usd": 83.60246, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T15-41-24__openai__codex__gpt-5.5-low__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.9072069505768408, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 7196, "avg_prompt_tokens": 383.2, "avg_response_tokens": 404.2, "avg_reasoning_tokens": null, "cost_usd": 167.475505, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T13-09-38__openai__cc__claude-opus-4-8__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.9013673265916536, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 2803.5, "avg_prompt_tokens": 172, "avg_response_tokens": 220.6, "avg_reasoning_tokens": 1.1, "cost_usd": 10.54558, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T15-05-05__vertex_ai__gemini-3-flash-preview__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8947443384133315, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 1819.1, "avg_prompt_tokens": 172.5, "avg_response_tokens": 198, "avg_reasoning_tokens": null, "cost_usd": 24.296021, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-06-35__vertex_ai__gemini-3.6-flash__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8933912548070075, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 2051.8, "avg_prompt_tokens": 172, "avg_response_tokens": 255.8, "avg_reasoning_tokens": null, "cost_usd": 35.952728, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T21-35-42__vertex_ai__gemini-3.5-flash" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8680387409200968, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 1061.1, "avg_prompt_tokens": 173, "avg_response_tokens": 210.7, "avg_reasoning_tokens": null, "cost_usd": 5.044435, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8656886483406923, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 12072.2, "avg_prompt_tokens": 185, "avg_response_tokens": 251.5, "avg_reasoning_tokens": null, "cost_usd": 1.583095, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-05-36__openai__gemma-4-31b-it-bf16__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8598490243555049, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 1507.4, "avg_prompt_tokens": 172.5, "avg_response_tokens": 273.6, "avg_reasoning_tokens": null, "cost_usd": 10.250173, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-01-49__vertex_ai__gemini-3.5-flash-lite__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8577837914826948, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 4390.6, "avg_prompt_tokens": 182.9, "avg_response_tokens": 561.4, "avg_reasoning_tokens": null, "cost_usd": 8.267917, "quantization": "fp8", "run_date": "2026-06-01", "run_id": "2026-06-01T00-16-47__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8515880928642644, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 2818.5, "avg_prompt_tokens": 190.8, "avg_response_tokens": 139.1, "avg_reasoning_tokens": null, "cost_usd": 0.911255, "quantization": "provider-internal", "run_date": "2026-06-23", "run_id": "2026-06-23T13-55-55__openrouter__deepseek__deepseek-v4-flash__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8473151972653468, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 1479.1, "avg_prompt_tokens": 172, "avg_response_tokens": 275.5, "avg_reasoning_tokens": null, "cost_usd": 10.396106, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T21-35-45__vertex_ai__gemini-2.5-flash" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8396952001139438, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 3506.7, "avg_prompt_tokens": 190.8, "avg_response_tokens": 139.3, "avg_reasoning_tokens": null, "cost_usd": 2.833558, "quantization": "provider-internal", "run_date": "2026-06-22", "run_id": "2026-06-22T13-03-10__openrouter__deepseek__deepseek-v4-pro__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.837202677681242, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 7765.3, "avg_prompt_tokens": 112.3, "avg_response_tokens": 221.7, "avg_reasoning_tokens": null, "cost_usd": 0.045241, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T08-43-34__openrouter__google__gemma-4-26b-a4b-it__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8367753881213502, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 5553.8, "avg_prompt_tokens": 223.4, "avg_response_tokens": 280.9, "avg_reasoning_tokens": null, "cost_usd": 1.136323, "quantization": "provider-internal", "run_date": "2026-05-28", "run_id": "2026-05-28T23-48-40__openrouter__tencent__hy3-preview" }, { "kind": "headline", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8312206238427574, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 4204.6, "avg_prompt_tokens": 277.7, "avg_response_tokens": 397.6, "avg_reasoning_tokens": null, "cost_usd": 31.9, "quantization": "provider-internal", "run_date": "2026-06-02", "run_id": "2026-06-02T14-33-20__openai__cc__claude-haiku-4-5-20251001__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7961116650049851, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 1723.1, "avg_prompt_tokens": 173, "avg_response_tokens": 506, "avg_reasoning_tokens": null, "cost_usd": 3.063079, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmmlu_de_gen" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7903432559464464, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 47379.9, "avg_prompt_tokens": 189, "avg_response_tokens": 261.2, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T17-59-01__openai__gemma4-12b-bf16__mmmlu_de_gen_nothink" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7880643782936904, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 13808.2, "avg_prompt_tokens": 186.9, "avg_response_tokens": 408.3, "avg_reasoning_tokens": null, "cost_usd": 1.122447, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T16-41-27__openai__qwen35-9b-bf16__mmmlu_de_gen_nothink" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmmlu_de_gen", "benchmark_display": "MMMLU (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.733941034040735, "items_effective": 14042, "items_total": 14042, "reasoning_off": true, "median_latency_ms": 6367.6, "avg_prompt_tokens": 212, "avg_response_tokens": 87.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T17-17-27__openai__qwen3-14b-bf16__mmmlu_de_gen_nothink" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8718428437792329, "items_effective": 11759, "items_total": 11759, "reasoning_off": false, "median_latency_ms": 29482.7, "avg_prompt_tokens": 1594.7, "avg_response_tokens": 145.5, "avg_reasoning_tokens": 137.3, "cost_usd": 193.51247, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8645292967089038, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 2232.6, "avg_prompt_tokens": 1664.9, "avg_response_tokens": 353.3, "avg_reasoning_tokens": null, "cost_usd": 66.75633, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8631686367888426, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 3261.4, "avg_prompt_tokens": 1664.9, "avg_response_tokens": 311.5, "avg_reasoning_tokens": 0.2, "cost_usd": 20.783733, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.859511863253678, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 1843.1, "avg_prompt_tokens": 1664.9, "avg_response_tokens": 259.2, "avg_reasoning_tokens": null, "cost_usd": 52.227652, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8215834679819712, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 1241.3, "avg_prompt_tokens": 1666.1, "avg_response_tokens": 304.5, "avg_reasoning_tokens": null, "cost_usd": 10.207914, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8209881792669445, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 64352.4, "avg_prompt_tokens": 1702.9, "avg_response_tokens": 502.5, "avg_reasoning_tokens": null, "cost_usd": 4.530131, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8076367038013437, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 3000, "avg_prompt_tokens": 288.1, "avg_response_tokens": 218.9, "avg_reasoning_tokens": null, "cost_usd": 3.713332, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8019389403860873, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 1570.4, "avg_prompt_tokens": 1664.9, "avg_response_tokens": 307.2, "avg_reasoning_tokens": null, "cost_usd": 14.903611, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.8004932392210222, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 4672.5, "avg_prompt_tokens": 1689.6, "avg_response_tokens": 984, "avg_reasoning_tokens": null, "cost_usd": 13.363754, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7962411769708309, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 2690.2, "avg_prompt_tokens": 1664.9, "avg_response_tokens": 786, "avg_reasoning_tokens": null, "cost_usd": 28.89494, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.781869, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 7766, "avg_prompt_tokens": 1700.6, "avg_response_tokens": 662.1, "avg_reasoning_tokens": null, "cost_usd": 4.641645, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "headline", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7526, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 3761, "avg_prompt_tokens": 2262, "avg_response_tokens": 433.7, "avg_reasoning_tokens": null, "cost_usd": 52.1, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7488732035036993, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 1982.2, "avg_prompt_tokens": 1718.7, "avg_response_tokens": 162.2, "avg_reasoning_tokens": null, "cost_usd": 1.041394, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7336508206480142, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 40409.9, "avg_prompt_tokens": 1693.6, "avg_response_tokens": 931.1, "avg_reasoning_tokens": null, "cost_usd": 3.633825, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "headline", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7315247895229187, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 6140.2, "avg_prompt_tokens": 1910.2, "avg_response_tokens": 586.3, "avg_reasoning_tokens": null, "cost_usd": 2.755966, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7314397482779148, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 92810.8, "avg_prompt_tokens": 1705.9, "avg_response_tokens": 493.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.712135385662046, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 2478.7, "avg_prompt_tokens": 1665.1, "avg_response_tokens": 1493.5, "avg_reasoning_tokens": null, "cost_usd": 8.95237, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6413810698188621, "items_effective": 11759, "items_total": 11759, "reasoning_off": true, "median_latency_ms": 29263.7, "avg_prompt_tokens": 1887.3, "avg_response_tokens": 413.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "headline", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7769784172661871, "items_effective": 139, "items_total": 139, "reasoning_off": false, "median_latency_ms": 4241.4, "avg_prompt_tokens": 107.1, "avg_response_tokens": 128.9, "avg_reasoning_tokens": 287.7, "cost_usd": 0.72467, "quantization": "provider-internal", "run_date": "2026-06-10", "run_id": "2026-06-10T08-40-06__gemini__gemini-3.1-pro-preview__include_de" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7482014388489209, "items_effective": 139, "items_total": 139, "reasoning_off": false, "median_latency_ms": 3994.8, "avg_prompt_tokens": 128.2, "avg_response_tokens": 50.2, "avg_reasoning_tokens": 103.4, "cost_usd": 0.70328, "quantization": "provider-internal", "run_date": "2026-06-12", "run_id": "2026-06-12T12-00-26__openai__codex__gpt-5.5-low__include_de" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7482014388489209, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 1367.9, "avg_prompt_tokens": 105.8, "avg_response_tokens": 168.7, "avg_reasoning_tokens": null, "cost_usd": 0.189399, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-40-53__vertex_ai__gemini-3.6-flash__include_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7410071942446043, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 1723.5, "avg_prompt_tokens": 107.1, "avg_response_tokens": 205.4, "avg_reasoning_tokens": null, "cost_usd": 0.279253, "quantization": "provider-internal", "run_date": "2026-06-10", "run_id": "2026-06-10T08-31-52__gemini__gemini-3.5-flash__include_de" }, { "kind": "headline", "model_slug": "minimax/minimax-m2.7", "model_display": "MiniMax M2.7", "model_vendor": "MiniMax", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7266187050359713, "items_effective": 139, "items_total": 139, "reasoning_off": false, "median_latency_ms": 12256.5, "avg_prompt_tokens": 151.1, "avg_response_tokens": -23.7, "avg_reasoning_tokens": 1467.5, "cost_usd": 0.287721, "quantization": "fp8", "run_date": "", "run_id": "2026-06-08T13-30-45__openrouter__minimax__minimax-m2.7__include_de" }, { "kind": "headline", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7266187050359713, "items_effective": 139, "items_total": 139, "reasoning_off": false, "median_latency_ms": 7722.8, "avg_prompt_tokens": 228.2, "avg_response_tokens": 437, "avg_reasoning_tokens": null, "cost_usd": 3.33239, "quantization": "provider-internal", "run_date": "2026-07-05", "run_id": "2026-07-05T21-52-59__openai__cc__claude-fable-5__include_de" }, { "kind": "headline", "model_slug": "anthropic/claude-sonnet-5", "model_display": "Sonnet 5", "model_vendor": "Anthropic", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7266187050359713, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 4924.9, "avg_prompt_tokens": 229.9, "avg_response_tokens": 263.5, "avg_reasoning_tokens": null, "cost_usd": 0.640572, "quantization": "provider-internal", "run_date": "2026-07-21", "run_id": "2026-07-21T11-51-21__openai__cc__claude-sonnet-5__include_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7194244604316546, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 894.8, "avg_prompt_tokens": 107.1, "avg_response_tokens": 151, "avg_reasoning_tokens": null, "cost_usd": 0.035204, "quantization": "provider-internal", "run_date": "2026-06-10", "run_id": "2026-06-10T08-32-28__gemini__gemini-3.1-flash-lite__include_de" }, { "kind": "headline", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7194244604316546, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 5767.2, "avg_prompt_tokens": 256.2, "avg_response_tokens": 343.7, "avg_reasoning_tokens": null, "cost_usd": 1.32297, "quantization": "provider-internal", "run_date": "2026-06-12", "run_id": "2026-06-12T12-12-42__openai__cc__claude-opus-4-8__include_de" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7194244604316546, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 2367.5, "avg_prompt_tokens": 106.4, "avg_response_tokens": 170.5, "avg_reasoning_tokens": null, "cost_usd": 0.07578, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__include_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.7-max", "model_display": "Qwen3.7 Max", "model_vendor": "Alibaba", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7122302158273381, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "include_de" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7050359712230215, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 1613, "avg_prompt_tokens": 105.8, "avg_response_tokens": 193.6, "avg_reasoning_tokens": null, "cost_usd": 0.070654, "quantization": "provider-internal", "run_date": "2026-06-02", "run_id": "2026-06-02T16-27-28__vertex_ai__gemini-2.5-flash__include_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7050359712230215, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 3444.4, "avg_prompt_tokens": 118.7, "avg_response_tokens": 121.3, "avg_reasoning_tokens": null, "cost_usd": 0.076764, "quantization": "fp8", "run_date": "2026-06-08", "run_id": "2026-06-08T16-40-45__openrouter__deepseek__deepseek-v4-pro__include_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7050359712230215, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 2534.1, "avg_prompt_tokens": 118.62589928057554, "avg_response_tokens": 111.28776978417268, "avg_reasoning_tokens": 0, "cost_usd": 0.006639779999999998, "quantization": "fp8", "run_date": "2026-06-08", "run_id": "2026-06-08T16-40-45__openrouter__deepseek__deepseek-v4-flash__include_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.7050359712230215, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 1442.6, "avg_prompt_tokens": 106.2, "avg_response_tokens": 235.4, "avg_reasoning_tokens": null, "cost_usd": 0.085618, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-40-53__vertex_ai__gemini-3.5-flash-lite__include_de" }, { "kind": "headline", "model_slug": "moonshotai/kimi-k2.6", "model_display": "Kimi K2.6", "model_vendor": "Moonshot", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.697841726618705, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 13652.4, "avg_prompt_tokens": 138.7, "avg_response_tokens": 527.1, "avg_reasoning_tokens": null, "cost_usd": 0.578522, "quantization": "fp8", "run_date": "2026-06-08", "run_id": "2026-06-08T12-01-03__openrouter__moonshotai__kimi-k2.6__include_de" }, { "kind": "headline", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6906474820143885, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 5476.3, "avg_prompt_tokens": 143.9, "avg_response_tokens": 222.4, "avg_reasoning_tokens": null, "cost_usd": 0.008633, "quantization": "provider-internal", "run_date": "2026-06-02", "run_id": "2026-06-02T15-51-31__openrouter__tencent__hy3-preview__include_de" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6906474820143885, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 27637.8, "avg_prompt_tokens": 123.1, "avg_response_tokens": 221.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-39-12__openai__gemma4-12b-bf16__include_de_nothink" }, { "kind": "headline", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6834532374100719, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 3454.5, "avg_prompt_tokens": 154.6, "avg_response_tokens": 347.4, "avg_reasoning_tokens": null, "cost_usd": 0.262946, "quantization": "provider-internal", "run_date": "2026-06-02", "run_id": "2026-06-02T07-34-47__openrouter__anthropic__claude-haiku-4.5__include_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6834532374100719, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 2722.1, "avg_prompt_tokens": 118.4, "avg_response_tokens": 545.1, "avg_reasoning_tokens": null, "cost_usd": 0.078233, "quantization": "fp8", "run_date": "2026-06-02", "run_id": "2026-06-02T19-43-02__openrouter__qwen__qwen3.6-35b-a3b__include_de" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6834532374100719, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 4560.8, "avg_prompt_tokens": 119.1, "avg_response_tokens": 205.4, "avg_reasoning_tokens": null, "cost_usd": 0.012265, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T13-34-38__openai__gemma-4-31b-it-bf16__include_de" }, { "kind": "headline", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6762589928057554, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-08", "run_id": "include_de" }, { "kind": "headline", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6762589928057554, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-08", "run_id": "include_de" }, { "kind": "headline", "model_slug": "openai/gpt-oss-120b", "model_display": "gpt-oss-120b", "model_vendor": "OpenAI", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6618705035971223, "items_effective": 139, "items_total": 139, "reasoning_off": false, "median_latency_ms": 1713.9, "avg_prompt_tokens": 154.1, "avg_response_tokens": 21.4, "avg_reasoning_tokens": 90.3, "cost_usd": 0.007814, "quantization": "", "run_date": "2026-05-29", "run_id": "2026-05-28T22-18-37__openrouter__openai__gpt-oss-120b" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6474820143884892, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 3713.4, "avg_prompt_tokens": 118.7, "avg_response_tokens": 275.8, "avg_reasoning_tokens": null, "cost_usd": 0.033463, "quantization": "bf16", "run_date": "2026-06-03", "run_id": "2026-06-03T19-31-33__openrouter__google__gemma-4-26b-a4b-it__include_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6474820143884892, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 10947.8, "avg_prompt_tokens": 122.4, "avg_response_tokens": 378, "avg_reasoning_tokens": null, "cost_usd": 0.009583, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-26-31__openai__qwen35-9b-bf16__include_de_nothink" }, { "kind": "headline", "model_slug": "x-ai/grok-4.3", "model_display": "grok-4.3", "model_vendor": "Unknown", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6330935251798561, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 603.6, "avg_prompt_tokens": 233.8, "avg_response_tokens": 20.5, "avg_reasoning_tokens": null, "cost_usd": 0.038129, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T17-36-44__openrouter__x-ai__grok-4.3__include_de" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.6330935251798561, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 3320.8, "avg_prompt_tokens": 135.4, "avg_response_tokens": 46, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-26-24__openai__qwen3-14b-bf16__include_de_nothink" }, { "kind": "headline", "model_slug": "mistralai/ministral-14b-2512", "model_display": "Ministral 14B", "model_vendor": "Mistral", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.5827338129496403, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 408.8, "avg_prompt_tokens": 109.1, "avg_response_tokens": 79.8, "avg_reasoning_tokens": null, "cost_usd": 0.01104, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T20-29-04__openrouter__mistralai__ministral-14b-2512__include_de" }, { "kind": "headline", "model_slug": "google/gemma-3-12b-it", "model_display": "gemma-3-12b-it", "model_vendor": "Unknown", "benchmark_key": "include_de", "benchmark_display": "INCLUDE (DE)", "subject": "", "metric": "exact_match,custom-extract", "score": 0.5323741007194245, "items_effective": 139, "items_total": 139, "reasoning_off": true, "median_latency_ms": 4999.9, "avg_prompt_tokens": 114.8, "avg_response_tokens": 160.5, "avg_reasoning_tokens": null, "cost_usd": 0.007335, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T20-29-04__openrouter__google__gemma-3-12b-it__include_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.7014689895519081, "items_effective": 1024, "items_total": 1024, "reasoning_off": false, "median_latency_ms": 3302.2, "avg_prompt_tokens": 625.7, "avg_response_tokens": 1.7, "avg_reasoning_tokens": 203.8, "cost_usd": 3.80609, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T16-39-20__vertex_ai__gemini-3.1-pro-preview__sb10k_de" }, { "kind": "headline", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6732610762561614, "items_effective": 1024, "items_total": 1024, "reasoning_off": false, "median_latency_ms": 4603.6, "avg_prompt_tokens": 1352.6, "avg_response_tokens": 19.3, "avg_reasoning_tokens": null, "cost_usd": 18.046495, "quantization": "provider-internal", "run_date": "2026-07-06", "run_id": "2026-07-06T09-59-46__openai__cc__claude-fable-5__sb10k_de" }, { "kind": "headline", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6452263951298309, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1875.1, "avg_prompt_tokens": 1372.7, "avg_response_tokens": 5, "avg_reasoning_tokens": null, "cost_usd": 7.15606, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T12-15-43__openai__cc__claude-opus-4-8__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6450889288835433, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 934.7, "avg_prompt_tokens": 625.7, "avg_response_tokens": 1.7, "avg_reasoning_tokens": null, "cost_usd": 0.974406, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-46-34__vertex_ai__gemini-3.6-flash__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6434209746954396, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 671.2, "avg_prompt_tokens": 650.6, "avg_response_tokens": 1.7, "avg_reasoning_tokens": null, "cost_usd": 1.019938, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-23-58__gemini__gemini-3.5-flash__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6357469684379895, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1042.7, "avg_prompt_tokens": 625.7, "avg_response_tokens": 1.7, "avg_reasoning_tokens": null, "cost_usd": 0.325672, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__sb10k_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.7-max", "model_display": "Qwen3.7 Max", "model_vendor": "Alibaba", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.631740667912767, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 985.3, "avg_prompt_tokens": 771.9, "avg_response_tokens": 1.7, "avg_reasoning_tokens": null, "cost_usd": 0.994736, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-50-42__openrouter__qwen__qwen3.7-max__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.62534070948932, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 588.5, "avg_prompt_tokens": 625.7, "avg_response_tokens": 1.7, "avg_reasoning_tokens": null, "cost_usd": 0.196583, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-46-34__vertex_ai__gemini-3.5-flash-lite__sb10k_de" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6245163585089542, "items_effective": 1024, "items_total": 1024, "reasoning_off": false, "median_latency_ms": 1360.3, "avg_prompt_tokens": 756.9, "avg_response_tokens": 5.9, "avg_reasoning_tokens": 2.2, "cost_usd": 4.12367, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T12-15-51__openai__codex__gpt-5.5-low__sb10k_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6237986747988183, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 543.6, "avg_prompt_tokens": 778.6, "avg_response_tokens": 2.7, "avg_reasoning_tokens": null, "cost_usd": 0.112399, "quantization": "bf16", "run_date": "2026-06-09", "run_id": "2026-06-09T14-38-22__openrouter__deepseek__deepseek-v4-flash__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.621653564215053, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 21451.4, "avg_prompt_tokens": 758.7, "avg_response_tokens": 2.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-40-46__openai__gemma4-12b-bf16__sb10k_de" }, { "kind": "headline", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6168613081997215, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 451.3, "avg_prompt_tokens": 1140.3, "avg_response_tokens": 2.8, "avg_reasoning_tokens": null, "cost_usd": 0.994569, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T14-50-42__openrouter__xiaomi__mimo-v2.5-pro__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6156808160819308, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 398.1, "avg_prompt_tokens": 650.7, "avg_response_tokens": 1.8, "avg_reasoning_tokens": null, "cost_usd": 0.204406, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-57-57__gemini__gemini-2.5-flash__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6102559401801054, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 5457.9, "avg_prompt_tokens": 758.7, "avg_response_tokens": 2.8, "avg_reasoning_tokens": null, "cost_usd": 0.094261, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-00-37__openai__gemma-4-31b-it-bf16__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.6036557749008397, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 459.7, "avg_prompt_tokens": 650.7, "avg_response_tokens": 1.8, "avg_reasoning_tokens": null, "cost_usd": 0.338577, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "2026-06-08T19-16-17__gemini__gemini-3.1-flash-lite__sb10k_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.5998355528595177, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1423.3, "avg_prompt_tokens": 778.6, "avg_response_tokens": 2.1, "avg_reasoning_tokens": null, "cost_usd": 1.28161, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T15-07-42__openrouter__deepseek__deepseek-v4-pro__sb10k_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.5838683580276286, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 451.2, "avg_prompt_tokens": 771.9, "avg_response_tokens": 2.7, "avg_reasoning_tokens": null, "cost_usd": 0.121349, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T16-50-36__openrouter__qwen__qwen3.6-35b-a3b__sb10k_de" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.5608252203057152, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 13255.3, "avg_prompt_tokens": 899.3, "avg_response_tokens": 2.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-27-27__openai__qwen3-14b-bf16__sb10k_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.5566743491349402, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 6958.5, "avg_prompt_tokens": 771.9, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": 0.079442, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-29-15__openai__qwen35-9b-bf16__sb10k_de" }, { "kind": "headline", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.3885762869763796, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 2348.8, "avg_prompt_tokens": 886.7, "avg_response_tokens": 2.7, "avg_reasoning_tokens": null, "cost_usd": 0.057818, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-57-56__openrouter__tencent__hy3-preview__sb10k_de" }, { "kind": "headline", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.23139053176220456, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1875, "avg_prompt_tokens": 765.6, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": 1.093541, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T14-57-57__openrouter__z-ai__glm-5.1__sb10k_de" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "sb10k_de", "benchmark_display": "SB10K (DE)", "subject": "", "metric": "mcc,none", "score": 0.15173246934981297, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 700.9, "avg_prompt_tokens": 758.8, "avg_response_tokens": 2.8, "avg_reasoning_tokens": null, "cost_usd": 0.116403, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "2026-06-08T16-39-47__vertex_ai__openai__google__gemma-4-26b-a4b-it-maas__sb10k_de" }, { "kind": "headline", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8774387193596799, "items_effective": 1024, "items_total": 1024, "reasoning_off": false, "median_latency_ms": 5521.2, "avg_prompt_tokens": 2083.6, "avg_response_tokens": 76.1, "avg_reasoning_tokens": null, "cost_usd": 30.069603, "quantization": "provider-internal", "run_date": "2026-07-06", "run_id": "2026-07-06T09-11-10__openai__cc__claude-fable-5__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.873323397913562, "items_effective": 1024, "items_total": 1024, "reasoning_off": false, "median_latency_ms": 4860.9, "avg_prompt_tokens": 1059.7, "avg_response_tokens": 29.5, "avg_reasoning_tokens": 406.7, "cost_usd": 10.036942, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-27-42__vertex_ai__gemini-3.1-pro-preview__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8615537848605578, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1044.3, "avg_prompt_tokens": 1060, "avg_response_tokens": 38.2, "avg_reasoning_tokens": null, "cost_usd": 1.921485, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-50-44__vertex_ai__gemini-3.6-flash__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8599703117268678, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 813, "avg_prompt_tokens": 1077, "avg_response_tokens": 24.6, "avg_reasoning_tokens": null, "cost_usd": 1.881132, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-27-05__gemini__gemini-3.5-flash__germeval_de" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8592739930382894, "items_effective": 1024, "items_total": 1024, "reasoning_off": false, "median_latency_ms": 1864.7, "avg_prompt_tokens": 1140.7, "avg_response_tokens": 29.1, "avg_reasoning_tokens": 11.3, "cost_usd": 7.080485, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T11-17-57__openai__codex__gpt-5.5-low__germeval_de" }, { "kind": "headline", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8584158415841584, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 2116.1, "avg_prompt_tokens": 2110.5, "avg_response_tokens": 42, "avg_reasoning_tokens": null, "cost_usd": 11.88141, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T11-17-44__openai__cc__claude-opus-4-8__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8538538538538539, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 701.8, "avg_prompt_tokens": 1060, "avg_response_tokens": 46.3, "avg_reasoning_tokens": null, "cost_usd": 0.44414, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-50-10__vertex_ai__gemini-3.5-flash-lite__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8472762645914396, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1196.5, "avg_prompt_tokens": 1060, "avg_response_tokens": 24.9, "avg_reasoning_tokens": null, "cost_usd": 0.619297, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8323809523809523, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 517.6, "avg_prompt_tokens": 1077, "avg_response_tokens": 25.7, "avg_reasoning_tokens": null, "cost_usd": 0.63029, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "2026-06-08T19-23-25__gemini__gemini-3.1-flash-lite__germeval_de" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8263821532492724, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 11921.4, "avg_prompt_tokens": 1153, "avg_response_tokens": 27.8, "avg_reasoning_tokens": null, "cost_usd": 0.151929, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T13-56-53__openai__gemma-4-31b-it-bf16__germeval_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8220623501199041, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1735.1, "avg_prompt_tokens": 1181.2, "avg_response_tokens": 26.3, "avg_reasoning_tokens": null, "cost_usd": 2.019523, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T15-18-51__openrouter__deepseek__deepseek-v4-pro__germeval_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.7-max", "model_display": "Qwen3.7 Max", "model_vendor": "Alibaba", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8207094918504315, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1639.6, "avg_prompt_tokens": 1154.3, "avg_response_tokens": 25.2, "avg_reasoning_tokens": null, "cost_usd": 1.574109, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-59-48__openrouter__qwen__qwen3.7-max__germeval_de" }, { "kind": "headline", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8186679630529898, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1089.5, "avg_prompt_tokens": 1533.5, "avg_response_tokens": 26.4, "avg_reasoning_tokens": null, "cost_usd": 1.486964, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T14-55-31__openrouter__xiaomi__mimo-v2.5-pro__germeval_de" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8177863702271629, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 1143.8, "avg_prompt_tokens": 1153, "avg_response_tokens": 30.3, "avg_reasoning_tokens": null, "cost_usd": 0.194537, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "2026-06-08T16-55-13__vertex_ai__openai__google__gemma-4-26b-a4b-it-maas__germeval_de" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.8177008177008178, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 515.9, "avg_prompt_tokens": 1077, "avg_response_tokens": 24.5, "avg_reasoning_tokens": null, "cost_usd": 0.393434, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-59-52__gemini__gemini-2.5-flash__germeval_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.801564027370479, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 768.8, "avg_prompt_tokens": 1181.2, "avg_response_tokens": 26.8, "avg_reasoning_tokens": null, "cost_usd": 0.177033, "quantization": "bf16", "run_date": "2026-06-09", "run_id": "2026-06-09T14-41-33__openrouter__deepseek__deepseek-v4-flash__germeval_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.7992424242424243, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 669.4, "avg_prompt_tokens": 1154.3, "avg_response_tokens": 27.3, "avg_reasoning_tokens": null, "cost_usd": 0.205209, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T17-07-57__openrouter__qwen__qwen3.6-35b-a3b__germeval_de" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.7963052989790956, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 43796.8, "avg_prompt_tokens": 1153, "avg_response_tokens": 28.1, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T19-56-46__openai__gemma4-12b-bf16__germeval_de" }, { "kind": "headline", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.7734597156398103, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 2501.5, "avg_prompt_tokens": 1313.7, "avg_response_tokens": 35.4, "avg_reasoning_tokens": null, "cost_usd": 0.093248, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T15-28-08__openrouter__tencent__hy3-preview__germeval_de" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.7300556586270872, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 21115.6, "avg_prompt_tokens": 1292.5, "avg_response_tokens": 28.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T19-56-53__openai__qwen3-14b-bf16__germeval_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.725982016090866, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 11765.4, "avg_prompt_tokens": 1154.3, "avg_response_tokens": 26.9, "avg_reasoning_tokens": null, "cost_usd": 0.122332, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T19-56-46__openai__qwen35-9b-bf16__germeval_de" }, { "kind": "headline", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "germeval_de", "benchmark_display": "GermEval NER (DE)", "subject": "", "metric": "micro_f1_no_misc,none", "score": 0.5217391304347826, "items_effective": 1024, "items_total": 1024, "reasoning_off": true, "median_latency_ms": 2195.1, "avg_prompt_tokens": 1127.6, "avg_response_tokens": 26.5, "avg_reasoning_tokens": null, "cost_usd": 1.708957, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T15-18-24__openrouter__z-ai__glm-5.1__germeval_de" }, { "kind": "headline", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.8359773634574729, "items_effective": 2048, "items_total": 2048, "reasoning_off": false, "median_latency_ms": 5195.4, "avg_prompt_tokens": 1661.5, "avg_response_tokens": 28, "avg_reasoning_tokens": null, "cost_usd": 44.765367, "quantization": "provider-internal", "run_date": "2026-07-06", "run_id": "2026-07-06T11-27-02__openai__cc__claude-fable-5__scala_de" }, { "kind": "headline", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.826171875, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 1997.4, "avg_prompt_tokens": 1684.8, "avg_response_tokens": 4, "avg_reasoning_tokens": null, "cost_usd": 17.45686, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T12-23-22__openai__cc__claude-opus-4-8__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.8042801216832116, "items_effective": 2048, "items_total": 2048, "reasoning_off": false, "median_latency_ms": 3710.8, "avg_prompt_tokens": 777.9, "avg_response_tokens": 1.5, "avg_reasoning_tokens": 260.5, "cost_usd": 10.366872, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-27-42__vertex_ai__gemini-3.1-pro-preview__scala_de" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7973679105362115, "items_effective": 2048, "items_total": 2048, "reasoning_off": false, "median_latency_ms": 1516.9, "avg_prompt_tokens": 913.2, "avg_response_tokens": 6, "avg_reasoning_tokens": 21.6, "cost_usd": 11.04651, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T12-23-33__openai__codex__gpt-5.5-low__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7943579103862382, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 789.1, "avg_prompt_tokens": 802.7, "avg_response_tokens": 1.5, "avg_reasoning_tokens": null, "cost_usd": 2.493871, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T14-31-33__gemini__gemini-3.5-flash__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7763764426835812, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 668.1, "avg_prompt_tokens": 777.7, "avg_response_tokens": 1.5, "avg_reasoning_tokens": null, "cost_usd": 2.412204, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-55-11__vertex_ai__gemini-3.6-flash__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7705081799070185, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 400.5, "avg_prompt_tokens": 802.7, "avg_response_tokens": 1.5, "avg_reasoning_tokens": null, "cost_usd": 0.500868, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T15-03-54__gemini__gemini-2.5-flash__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.769742695187095, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 944.5, "avg_prompt_tokens": 777.7, "avg_response_tokens": 1.5, "avg_reasoning_tokens": null, "cost_usd": 0.805663, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7512004710561102, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 562.8, "avg_prompt_tokens": 777.7, "avg_response_tokens": 1.5, "avg_reasoning_tokens": null, "cost_usd": 0.485568, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-53-30__vertex_ai__gemini-3.5-flash-lite__scala_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7486240818543228, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 533.1, "avg_prompt_tokens": 949.8, "avg_response_tokens": 2.5, "avg_reasoning_tokens": null, "cost_usd": 0.273794, "quantization": "bf16", "run_date": "2026-06-09", "run_id": "2026-06-09T14-49-50__openrouter__deepseek__deepseek-v4-flash__scala_de" }, { "kind": "headline", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7480495842012994, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 573.3, "avg_prompt_tokens": 1301, "avg_response_tokens": 2.5, "avg_reasoning_tokens": null, "cost_usd": 2.326171, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T15-24-43__openrouter__xiaomi__mimo-v2.5-pro__scala_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7420709907639391, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 455.9, "avg_prompt_tokens": 802.7, "avg_response_tokens": 1.5, "avg_reasoning_tokens": null, "cost_usd": 0.831407, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "2026-06-08T19-17-57__gemini__gemini-3.1-flash-lite__scala_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7349109392827395, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 1407.8, "avg_prompt_tokens": 949.8, "avg_response_tokens": 1.6, "avg_reasoning_tokens": null, "cost_usd": 3.117131, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T15-39-20__openrouter__deepseek__deepseek-v4-pro__scala_de" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.7101242540750777, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 1282.1, "avg_prompt_tokens": 910.7, "avg_response_tokens": 2.5, "avg_reasoning_tokens": null, "cost_usd": 0.225657, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T13-36-12__openai__gemma-4-31b-it-bf16__scala_de" }, { "kind": "headline", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6990153052359315, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 1692.4, "avg_prompt_tokens": 901.6, "avg_response_tokens": 2.5, "avg_reasoning_tokens": null, "cost_usd": 2.570165, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T15-44-40__openrouter__z-ai__glm-5.1__scala_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.7-max", "model_display": "Qwen3.7 Max", "model_vendor": "Alibaba", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.697938584106779, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 1021, "avg_prompt_tokens": 928.9, "avg_response_tokens": 1.6, "avg_reasoning_tokens": null, "cost_usd": 2.390335, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T15-15-22__openrouter__qwen__qwen3.7-max__scala_de" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6630750414295619, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 458.1, "avg_prompt_tokens": 910.7, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": 0.280416, "quantization": "provider-internal", "run_date": "2026-06-08", "run_id": "2026-06-08T16-45-24__vertex_ai__openai__google__gemma-4-26b-a4b-it-maas__scala_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6424793088203536, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 505.2, "avg_prompt_tokens": 928.9, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": 0.290741, "quantization": "fp8", "run_date": "2026-06-09", "run_id": "2026-06-09T16-55-32__openrouter__qwen__qwen3.6-35b-a3b__scala_de" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6406445161763898, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 27849.5, "avg_prompt_tokens": 910.7, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-44-14__openai__gemma4-12b-bf16__scala_de" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6396556632627416, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 15713.9, "avg_prompt_tokens": 1060, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-29-49__openai__qwen3-14b-bf16__scala_de" }, { "kind": "headline", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6245821267107304, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 2480.8, "avg_prompt_tokens": 1027.6, "avg_response_tokens": 2.6, "avg_reasoning_tokens": null, "cost_usd": 0.133381, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T15-59-54__openrouter__tencent__hy3-preview__scala_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "scala_de", "benchmark_display": "ScaLA (DE)", "subject": "", "metric": "mcc,none", "score": 0.6206195533916511, "items_effective": 2048, "items_total": 2048, "reasoning_off": true, "median_latency_ms": 8396.7, "avg_prompt_tokens": 928.9, "avg_response_tokens": 2.5, "avg_reasoning_tokens": null, "cost_usd": 0.191007, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-30-49__openai__qwen35-9b-bf16__scala_de" }, { "kind": "headline", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8953900709219859, "items_effective": 564, "items_total": 564, "reasoning_off": false, "median_latency_ms": 20284.3, "avg_prompt_tokens": 3008.8, "avg_response_tokens": 1338.2, "avg_reasoning_tokens": null, "cost_usd": 54.70676, "quantization": "provider-internal", "run_date": "2026-07-05", "run_id": "2026-07-05T22-33-14__openai__cc__claude-fable-5__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8812056737588653, "items_effective": 564, "items_total": 564, "reasoning_off": false, "median_latency_ms": 12140, "avg_prompt_tokens": 1464.9, "avg_response_tokens": 510.8, "avg_reasoning_tokens": 795.7, "cost_usd": 10.494854, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-30-10__vertex_ai__gemini-3.1-pro-preview__musr_de" }, { "kind": "headline", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8634751773049646, "items_effective": 564, "items_total": 564, "reasoning_off": false, "median_latency_ms": 13579.3, "avg_prompt_tokens": 1449.5, "avg_response_tokens": 414.8, "avg_reasoning_tokens": 202.4, "cost_usd": 14.530875, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-13__openai__codex__gpt-5.5-low__musr_de" }, { "kind": "headline", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8634751773049646, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 13872, "avg_prompt_tokens": 3039.8, "avg_response_tokens": 1075.6, "avg_reasoning_tokens": null, "cost_usd": 23.738825, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-02__openai__cc__claude-opus-4-8__musr_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8528368794326241, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 12597.3, "avg_prompt_tokens": 1657.8, "avg_response_tokens": 719.7, "avg_reasoning_tokens": null, "cost_usd": 0.744786, "quantization": "provider-internal", "run_date": "2026-06-23", "run_id": "2026-06-23T13-55-55__openrouter__deepseek__deepseek-v4-pro__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8439716312056738, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 2950, "avg_prompt_tokens": 1467.3, "avg_response_tokens": 568.5, "avg_reasoning_tokens": null, "cost_usd": 0.731757, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-44-23__vertex_ai__gemini-3.1-flash-lite__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8421985815602837, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 5791.7, "avg_prompt_tokens": 1464.9, "avg_response_tokens": 850.2, "avg_reasoning_tokens": null, "cost_usd": 5.554927, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-54-48__vertex_ai__gemini-3.5-flash__musr_de" }, { "kind": "headline", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8368794326241135, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "headline", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8351063829787234, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 10176.2, "avg_prompt_tokens": 1659.8, "avg_response_tokens": 798.9, "avg_reasoning_tokens": null, "cost_usd": 0.256542, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "2026-06-10T11-50-02__openrouter__deepseek__deepseek-v4-flash__musr_de" }, { "kind": "headline", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8351063829787234, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 23295.5, "avg_prompt_tokens": 1477.9, "avg_response_tokens": 650.1, "avg_reasoning_tokens": null, "cost_usd": 0.232021, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-02-37__openai__gemma-4-31b-it-bf16__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8333333333333334, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 6278.9, "avg_prompt_tokens": 1464.9, "avg_response_tokens": 1077.4, "avg_reasoning_tokens": null, "cost_usd": 1.766489, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-10-07__vertex_ai__gemini-2.5-flash__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8173758865248227, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 2855.5, "avg_prompt_tokens": 1464.9, "avg_response_tokens": 649.3, "avg_reasoning_tokens": null, "cost_usd": 1.163382, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-57-34__vertex_ai__gemini-3.5-flash-lite__musr_de" }, { "kind": "headline", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8156028368794326, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 195497.7, "avg_prompt_tokens": 1477.9, "avg_response_tokens": 707.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-52-24__openai__gemma4-12b-bf16__musr_de" }, { "kind": "headline", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8138297872340425, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "headline", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8085106382978723, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "headline", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8085106382978723, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 61141.4, "avg_prompt_tokens": 1469.2, "avg_response_tokens": 1629.7, "avg_reasoning_tokens": null, "cost_usd": 0.220736, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-33-49__openai__qwen35-9b-bf16__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.8031914893617021, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 4008.2, "avg_prompt_tokens": 1464.9, "avg_response_tokens": 663.7, "avg_reasoning_tokens": null, "cost_usd": 4.046994, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-59-41__vertex_ai__gemini-3.6-flash__musr_de" }, { "kind": "headline", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.7925531914893617, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 5411.2, "avg_prompt_tokens": 1464.9, "avg_response_tokens": 622.9, "avg_reasoning_tokens": null, "cost_usd": 1.467145, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__musr_de" }, { "kind": "headline", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "", "metric": "acc,none", "score": 0.6968085106382979, "items_effective": 564, "items_total": 564, "reasoning_off": true, "median_latency_ms": 263430.7, "avg_prompt_tokens": 1728.9, "avg_response_tokens": 1270.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-34-46__openai__qwen3-14b-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.9288702928870292, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 1880.7, "avg_prompt_tokens": 1834.7, "avg_response_tokens": 229.5, "avg_reasoning_tokens": null, "cost_usd": 3.454245, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.9260808926080892, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 2811.5, "avg_prompt_tokens": 1834.7, "avg_response_tokens": 258.2, "avg_reasoning_tokens": null, "cost_usd": 1.213218, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.9260808926080892, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 1957.9, "avg_prompt_tokens": 1834.7, "avg_response_tokens": 163.5, "avg_reasoning_tokens": null, "cost_usd": 2.85214, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.9232914923291492, "items_effective": 717, "items_total": 717, "reasoning_off": false, "median_latency_ms": 13410.6, "avg_prompt_tokens": 1790.1, "avg_response_tokens": 128.6, "avg_reasoning_tokens": 81.5, "cost_usd": 10.93581, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.9121338912133892, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 53750.4, "avg_prompt_tokens": 1872.7, "avg_response_tokens": 419.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.902370990237099, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 371.5, "avg_response_tokens": 133.5, "avg_reasoning_tokens": null, "cost_usd": 0.199182, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8967921896792189, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 4012.8, "avg_prompt_tokens": 1859, "avg_response_tokens": 662, "avg_reasoning_tokens": null, "cost_usd": 0.599103, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8940027894002789, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 1204.8, "avg_prompt_tokens": 1836.3, "avg_response_tokens": 240.9, "avg_reasoning_tokens": null, "cost_usd": 0.584118, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8884239888423989, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 4370.2, "avg_prompt_tokens": 1834.7, "avg_response_tokens": 750.9, "avg_reasoning_tokens": null, "cost_usd": 1.740074, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8842398884239888, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 1817.8, "avg_prompt_tokens": 1899.6, "avg_response_tokens": 101.1, "avg_reasoning_tokens": null, "cost_usd": 0.059525, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8814504881450488, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 1333.8, "avg_prompt_tokens": 1834.7, "avg_response_tokens": 209.8, "avg_reasoning_tokens": null, "cost_usd": 0.770748, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.873082, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 7359.6, "avg_prompt_tokens": 1878.4, "avg_response_tokens": 576.8, "avg_reasoning_tokens": null, "cost_usd": 0.195273, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8675034867503487, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 2361.9, "avg_prompt_tokens": 1835.7, "avg_response_tokens": 919.2, "avg_reasoning_tokens": null, "cost_usd": 0.395247, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8591352859135286, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "biology", "metric": "exact_match,custom-extract", "score": 0.8577405857740585, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 7501.5, "avg_prompt_tokens": 2158, "avg_response_tokens": 559.6, "avg_reasoning_tokens": null, "cost_usd": 0.172726, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.9112801013941698, "items_effective": 789, "items_total": 789, "reasoning_off": false, "median_latency_ms": 32144.9, "avg_prompt_tokens": 1318.4, "avg_response_tokens": 113.6, "avg_reasoning_tokens": 132.7, "cost_usd": 11.03268, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.8960709759188846, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 2298.5, "avg_prompt_tokens": 1393.1, "avg_response_tokens": 391.1, "avg_reasoning_tokens": null, "cost_usd": 4.426061, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.8948035487959443, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 3054.2, "avg_prompt_tokens": 1393.1, "avg_response_tokens": 327.2, "avg_reasoning_tokens": null, "cost_usd": 1.324018, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.8821292775665399, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 2207.6, "avg_prompt_tokens": 1393.1, "avg_response_tokens": 294.8, "avg_reasoning_tokens": null, "cost_usd": 3.393201, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.8745247148288974, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 70487.6, "avg_prompt_tokens": 1431.1, "avg_response_tokens": 509.1, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.85297845373891, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 1229.8, "avg_prompt_tokens": 1394.2, "avg_response_tokens": 307.6, "avg_reasoning_tokens": null, "cost_usd": 0.634966, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.85297845373891, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 210, "avg_response_tokens": 319.7, "avg_reasoning_tokens": null, "cost_usd": 0.29151, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.844106463878327, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 1549.2, "avg_prompt_tokens": 1393.1, "avg_response_tokens": 355.2, "avg_reasoning_tokens": null, "cost_usd": 1.030424, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.839037, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 6555.6, "avg_prompt_tokens": 1431.5, "avg_response_tokens": 718.4, "avg_reasoning_tokens": null, "cost_usd": 0.318803, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.835234474017744, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 2273.2, "avg_prompt_tokens": 1393.1, "avg_response_tokens": 746.5, "avg_reasoning_tokens": null, "cost_usd": 1.802294, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.835234474017744, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 4160.3, "avg_prompt_tokens": 1404.4, "avg_response_tokens": 1045.5, "avg_reasoning_tokens": null, "cost_usd": 0.907998, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.7896070975918885, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.7832699619771863, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 6656.8, "avg_prompt_tokens": 1662, "avg_response_tokens": 500.7, "avg_reasoning_tokens": null, "cost_usd": 0.159964, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.7769328263624842, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 1932.9, "avg_prompt_tokens": 1393.5, "avg_response_tokens": 1436.6, "avg_reasoning_tokens": null, "cost_usd": 0.56264, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "business", "metric": "exact_match,custom-extract", "score": 0.761723700887199, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 1302.1, "avg_prompt_tokens": 1485, "avg_response_tokens": 134.3, "avg_reasoning_tokens": null, "cost_usd": 0.054857, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8984098939929329, "items_effective": 1132, "items_total": 1132, "reasoning_off": false, "median_latency_ms": 38341.1, "avg_prompt_tokens": 1740.3, "avg_response_tokens": 258.5, "avg_reasoning_tokens": 157.1, "cost_usd": 23.963365, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.892226148409894, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 3934.5, "avg_prompt_tokens": 1801, "avg_response_tokens": 514.6, "avg_reasoning_tokens": null, "cost_usd": 2.766764, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8913427561837456, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 3124.3, "avg_prompt_tokens": 1801, "avg_response_tokens": 572.8, "avg_reasoning_tokens": null, "cost_usd": 8.894004, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8833922261484098, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 2343.1, "avg_prompt_tokens": 1801, "avg_response_tokens": 396.4, "avg_reasoning_tokens": null, "cost_usd": 6.423755, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8816254416961131, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 6067.8, "avg_prompt_tokens": 1861.8, "avg_response_tokens": 1248.3, "avg_reasoning_tokens": null, "cost_usd": 1.609929, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8710247349823321, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 1629, "avg_prompt_tokens": 1802.3, "avg_response_tokens": 463.7, "avg_reasoning_tokens": null, "cost_usd": 1.291743, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8692579505300353, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 3967, "avg_prompt_tokens": 1801, "avg_response_tokens": 1148, "avg_reasoning_tokens": null, "cost_usd": 3.858767, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8666077738515902, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 90642.9, "avg_prompt_tokens": 1839, "avg_response_tokens": 642.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.8666077738515902, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 255.4, "avg_response_tokens": 317.3, "avg_reasoning_tokens": null, "cost_usd": 0.438281, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.85424, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 8321.8, "avg_prompt_tokens": 1839.2, "avg_response_tokens": 806.1, "avg_reasoning_tokens": null, "cost_usd": 0.530739, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.838339222614841, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 1922.1, "avg_prompt_tokens": 1801, "avg_response_tokens": 509.8, "avg_reasoning_tokens": null, "cost_usd": 2.054206, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.803886925795053, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.7835689045936396, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 3504.2, "avg_prompt_tokens": 1801.4, "avg_response_tokens": 2285.9, "avg_reasoning_tokens": null, "cost_usd": 1.223667, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.7561837455830389, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 6562.5, "avg_prompt_tokens": 1937, "avg_response_tokens": 719.7, "avg_reasoning_tokens": null, "cost_usd": 0.301255, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "chemistry", "metric": "exact_match,custom-extract", "score": 0.7332155477031802, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 2611, "avg_prompt_tokens": 1790.2, "avg_response_tokens": 243.3, "avg_reasoning_tokens": null, "cost_usd": 0.121773, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.9, "items_effective": 410, "items_total": 410, "reasoning_off": false, "median_latency_ms": 24086.4, "avg_prompt_tokens": 1765.7, "avg_response_tokens": 116.2, "avg_reasoning_tokens": 122.7, "cost_usd": 6.558625, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8878048780487805, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 1339.7, "avg_prompt_tokens": 1856.4, "avg_response_tokens": 306.3, "avg_reasoning_tokens": null, "cost_usd": 0.374047, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8853658536585366, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 2036.9, "avg_prompt_tokens": 1854.9, "avg_response_tokens": 217.3, "avg_reasoning_tokens": null, "cost_usd": 1.808931, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.875609756097561, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 2204.4, "avg_prompt_tokens": 1854.9, "avg_response_tokens": 314.3, "avg_reasoning_tokens": null, "cost_usd": 2.30045, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8658536585365854, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 3137.3, "avg_prompt_tokens": 1854.9, "avg_response_tokens": 318.6, "avg_reasoning_tokens": null, "cost_usd": 0.772099, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8585365853658536, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 2133.9, "avg_prompt_tokens": 1905.8, "avg_response_tokens": 189.4, "avg_reasoning_tokens": null, "cost_usd": 0.037169, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8585365853658536, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 63917.4, "avg_prompt_tokens": 1892.9, "avg_response_tokens": 485, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8536585365853658, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 259.4, "avg_response_tokens": 230.5, "avg_reasoning_tokens": null, "cost_usd": 0.128498, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8512195121951219, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 3495.8, "avg_prompt_tokens": 1854.9, "avg_response_tokens": 868, "avg_reasoning_tokens": null, "cost_usd": 1.116755, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8512195121951219, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 4621.3, "avg_prompt_tokens": 1875.6, "avg_response_tokens": 1024.1, "avg_reasoning_tokens": null, "cost_usd": 0.492133, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.839024, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 20700.8, "avg_prompt_tokens": 1893.8, "avg_response_tokens": 600.6, "avg_reasoning_tokens": null, "cost_usd": 0.161554, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8341463414634146, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.8317073170731707, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 1500.1, "avg_prompt_tokens": 1854.9, "avg_response_tokens": 281.3, "avg_reasoning_tokens": null, "cost_usd": 0.516448, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.7780487804878049, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 2719.4, "avg_prompt_tokens": 1855.9, "avg_response_tokens": 1541.1, "avg_reasoning_tokens": null, "cost_usd": 0.32884, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "computer_science", "metric": "exact_match,custom-extract", "score": 0.7682926829268293, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 6936, "avg_prompt_tokens": 2092.8, "avg_response_tokens": 810.2, "avg_reasoning_tokens": null, "cost_usd": 0.126899, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8921800947867299, "items_effective": 844, "items_total": 844, "reasoning_off": false, "median_latency_ms": 19598.1, "avg_prompt_tokens": 1505, "avg_response_tokens": 92.8, "avg_reasoning_tokens": 78.3, "cost_usd": 10.68407, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8909952606635071, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 2082.2, "avg_prompt_tokens": 1578.7, "avg_response_tokens": 249.1, "avg_reasoning_tokens": null, "cost_usd": 3.89094, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8874407582938388, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 2580.3, "avg_prompt_tokens": 1578.7, "avg_response_tokens": 240.3, "avg_reasoning_tokens": 0.1, "cost_usd": 1.274939, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8874407582938388, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 1726.4, "avg_prompt_tokens": 1578.7, "avg_response_tokens": 196.2, "avg_reasoning_tokens": null, "cost_usd": 3.240467, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8732227488151659, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 1122.7, "avg_prompt_tokens": 1580, "avg_response_tokens": 228.4, "avg_reasoning_tokens": null, "cost_usd": 0.618855, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8684834123222749, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 55984.4, "avg_prompt_tokens": 1616.7, "avg_response_tokens": 435.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8637440758293838, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 2628.4, "avg_prompt_tokens": 1578.7, "avg_response_tokens": 479, "avg_reasoning_tokens": null, "cost_usd": 1.409783, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8590047393364929, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 1330, "avg_prompt_tokens": 1578.7, "avg_response_tokens": 212, "avg_reasoning_tokens": null, "cost_usd": 0.847078, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8530805687203792, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 4123.7, "avg_prompt_tokens": 1588.9, "avg_response_tokens": 713.2, "avg_reasoning_tokens": null, "cost_usd": 0.714185, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8412322274881516, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 258.5, "avg_response_tokens": 143.9, "avg_reasoning_tokens": null, "cost_usd": 0.200559, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.825829, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 44116.7, "avg_prompt_tokens": 1619.3, "avg_response_tokens": 579.8, "avg_reasoning_tokens": null, "cost_usd": 0.315522, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8234597156398105, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 5903.5, "avg_prompt_tokens": 1841.7, "avg_response_tokens": 498.8, "avg_reasoning_tokens": null, "cost_usd": 0.176479, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.8151658767772512, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.7914691943127962, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 2372.8, "avg_prompt_tokens": 1579.7, "avg_response_tokens": 939.2, "avg_reasoning_tokens": null, "cost_usd": 0.449863, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "economics", "metric": "exact_match,custom-extract", "score": 0.75, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 1361.6, "avg_prompt_tokens": 1640.2, "avg_response_tokens": 57.1, "avg_reasoning_tokens": null, "cost_usd": 0.046373, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.8328173374613003, "items_effective": 969, "items_total": 969, "reasoning_off": false, "median_latency_ms": 51148.5, "avg_prompt_tokens": 1428.4, "avg_response_tokens": 255.5, "avg_reasoning_tokens": 311, "cost_usd": 23.38808, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.8245614035087719, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 3624.9, "avg_prompt_tokens": 1546.2, "avg_response_tokens": 656.2, "avg_reasoning_tokens": null, "cost_usd": 7.970271, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.8121775025799793, "items_effective": 969, "items_total": 969, "reasoning_off": false, "median_latency_ms": 4317.6, "avg_prompt_tokens": 1546.2, "avg_response_tokens": 548.2, "avg_reasoning_tokens": 0.6, "cost_usd": 2.344556, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.804953560371517, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 2743.4, "avg_prompt_tokens": 1546.2, "avg_response_tokens": 453.6, "avg_reasoning_tokens": null, "cost_usd": 5.543898, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.7781217750257998, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 1897.7, "avg_prompt_tokens": 1548.5, "avg_response_tokens": 507.3, "avg_reasoning_tokens": null, "cost_usd": 1.106683, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.7750257997936016, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 12344.5, "avg_prompt_tokens": 1566.2, "avg_response_tokens": 2227.7, "avg_reasoning_tokens": null, "cost_usd": 2.284209, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.7708978328173375, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 114942.1, "avg_prompt_tokens": 1584.3, "avg_response_tokens": 745.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.7420020639834881, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 337.8, "avg_response_tokens": 420.1, "avg_reasoning_tokens": null, "cost_usd": 0.496532, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.734778, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 13008.2, "avg_prompt_tokens": 1584.4, "avg_response_tokens": 929.8, "avg_reasoning_tokens": null, "cost_usd": 0.488739, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.7275541795665634, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 1913.1, "avg_prompt_tokens": 1546.2, "avg_response_tokens": 462.2, "avg_reasoning_tokens": null, "cost_usd": 1.569214, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.7162022703818369, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 5889.1, "avg_prompt_tokens": 1546.3, "avg_response_tokens": 2263.7, "avg_reasoning_tokens": null, "cost_usd": 5.932759, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.6470588235294118, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.6439628482972136, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 9745.4, "avg_prompt_tokens": 1610.9, "avg_response_tokens": 1277.8, "avg_reasoning_tokens": null, "cost_usd": 0.390615, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.6408668730650154, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 3125.4, "avg_prompt_tokens": 1489, "avg_response_tokens": 309.9, "avg_reasoning_tokens": null, "cost_usd": 0.132385, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "engineering", "metric": "exact_match,custom-extract", "score": 0.5841073271413829, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 9421.1, "avg_prompt_tokens": 1547.3, "avg_response_tokens": 3880.5, "avg_reasoning_tokens": null, "cost_usd": 1.654007, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.8180494905385735, "items_effective": 687, "items_total": 687, "reasoning_off": false, "median_latency_ms": 25847.2, "avg_prompt_tokens": 1279.9, "avg_response_tokens": 85.6, "avg_reasoning_tokens": 115.2, "cost_usd": 8.533945, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.8005822416302766, "items_effective": 687, "items_total": 687, "reasoning_off": false, "median_latency_ms": 2658.6, "avg_prompt_tokens": 1315.1, "avg_response_tokens": 209.6, "avg_reasoning_tokens": 0.8, "cost_usd": 0.885459, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7933042212518195, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 1746.6, "avg_prompt_tokens": 1315.1, "avg_response_tokens": 171.3, "avg_reasoning_tokens": null, "cost_usd": 2.237972, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7860262008733624, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 2024.5, "avg_prompt_tokens": 1315.1, "avg_response_tokens": 228.4, "avg_reasoning_tokens": null, "cost_usd": 2.767079, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7481804949053857, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 997.9, "avg_prompt_tokens": 1316.6, "avg_response_tokens": 185, "avg_reasoning_tokens": null, "cost_usd": 0.413757, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.74235807860262, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 253.9, "avg_response_tokens": 122.8, "avg_reasoning_tokens": null, "cost_usd": 0.149247, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7409024745269287, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 53822.2, "avg_prompt_tokens": 1353.1, "avg_response_tokens": 429.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7394468704512372, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 1498.2, "avg_prompt_tokens": 1315.1, "avg_response_tokens": 164.5, "avg_reasoning_tokens": null, "cost_usd": 0.55364, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7365356622998545, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 5575, "avg_prompt_tokens": 1611.2, "avg_response_tokens": 443.4, "avg_reasoning_tokens": null, "cost_usd": 0.128257, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7292576419213974, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.727802037845706, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 5142.5, "avg_prompt_tokens": 1346.9, "avg_response_tokens": 657.9, "avg_reasoning_tokens": null, "cost_usd": 0.51845, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7248908296943232, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 1818.8, "avg_prompt_tokens": 1315.1, "avg_response_tokens": 371, "avg_reasoning_tokens": null, "cost_usd": 0.908228, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.7132459970887919, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 2074.2, "avg_prompt_tokens": 1403, "avg_response_tokens": 120.6, "avg_reasoning_tokens": null, "cost_usd": 0.049266, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.71179, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 6875.6, "avg_prompt_tokens": 1353.3, "avg_response_tokens": 575.1, "avg_reasoning_tokens": null, "cost_usd": 0.231873, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "health", "metric": "exact_match,custom-extract", "score": 0.6652110625909753, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 1702.6, "avg_prompt_tokens": 1316.1, "avg_response_tokens": 634.2, "avg_reasoning_tokens": null, "cost_usd": 0.264701, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.8083989501312336, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 2508.3, "avg_prompt_tokens": 2770.4, "avg_response_tokens": 158.6, "avg_reasoning_tokens": null, "cost_usd": 0.709096, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.8057742782152231, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 1977.5, "avg_prompt_tokens": 2770.4, "avg_response_tokens": 194.4, "avg_reasoning_tokens": null, "cost_usd": 2.249967, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.800524934383202, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 1562.4, "avg_prompt_tokens": 2770.4, "avg_response_tokens": 120.6, "avg_reasoning_tokens": null, "cost_usd": 1.927933, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7769028871391076, "items_effective": 381, "items_total": 381, "reasoning_off": false, "median_latency_ms": 24273.6, "avg_prompt_tokens": 2691.6, "avg_response_tokens": 80.4, "avg_reasoning_tokens": 88, "cost_usd": 7.051285, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7506561679790026, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 1527.6, "avg_prompt_tokens": 2770.4, "avg_response_tokens": 163.7, "avg_reasoning_tokens": null, "cost_usd": 0.472555, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7454068241469817, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 1073.1, "avg_prompt_tokens": 2774.4, "avg_response_tokens": 189, "avg_reasoning_tokens": null, "cost_usd": 0.367411, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7454068241469817, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 62387.9, "avg_prompt_tokens": 2808.4, "avg_response_tokens": 419.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7191601049868767, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 498.4, "avg_response_tokens": 103, "avg_reasoning_tokens": null, "cost_usd": 0.116749, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7139107611548556, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 4854.9, "avg_prompt_tokens": 3413.4, "avg_response_tokens": 299.2, "avg_reasoning_tokens": null, "cost_usd": 0.090787, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.7086614173228346, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 1366.9, "avg_prompt_tokens": 2770.4, "avg_response_tokens": 257.6, "avg_reasoning_tokens": null, "cost_usd": 0.496573, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.6902887139107612, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 5355.8, "avg_prompt_tokens": 2793.8, "avg_response_tokens": 688.6, "avg_reasoning_tokens": null, "cost_usd": 0.341966, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.6824146981627297, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 1881.7, "avg_prompt_tokens": 3036.7, "avg_response_tokens": 75.7, "avg_reasoning_tokens": null, "cost_usd": 0.035727, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.666667, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 10074.1, "avg_prompt_tokens": 2808.7, "avg_response_tokens": 560.6, "avg_reasoning_tokens": null, "cost_usd": 0.169959, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.6535433070866141, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "history", "metric": "exact_match,custom-extract", "score": 0.5931758530183727, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 2364.7, "avg_prompt_tokens": 2771.4, "avg_response_tokens": 628, "avg_reasoning_tokens": null, "cost_usd": 0.201305, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.7476538060479666, "items_effective": 959, "items_total": 959, "reasoning_off": false, "median_latency_ms": 37201.7, "avg_prompt_tokens": 1925.8, "avg_response_tokens": 118, "avg_reasoning_tokens": 186.5, "cost_usd": 17.99402, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.7320125130344108, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 2656.6, "avg_prompt_tokens": 1987.2, "avg_response_tokens": 211.4, "avg_reasoning_tokens": null, "cost_usd": 1.561187, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.7320125130344108, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 1838.1, "avg_prompt_tokens": 1987.2, "avg_response_tokens": 212.8, "avg_reasoning_tokens": null, "cost_usd": 4.388934, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.7278415015641293, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 2334.7, "avg_prompt_tokens": 1987.2, "avg_response_tokens": 283.7, "avg_reasoning_tokens": null, "cost_usd": 5.307169, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.6277372262773723, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 1650, "avg_prompt_tokens": 1988, "avg_response_tokens": 319, "avg_reasoning_tokens": null, "cost_usd": 0.930566, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.5901981230448383, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 87325.7, "avg_prompt_tokens": 2025.2, "avg_response_tokens": 537.1, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.5818561001042752, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 1732.2, "avg_prompt_tokens": 1987.2, "avg_response_tokens": 265, "avg_reasoning_tokens": null, "cost_usd": 1.206971, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.5526590198123045, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 551.9, "avg_response_tokens": 170.9, "avg_reasoning_tokens": null, "cost_usd": 0.372852, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.5401459854014599, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 1724, "avg_prompt_tokens": 1987.2, "avg_response_tokens": 442.6, "avg_reasoning_tokens": null, "cost_usd": 1.627871, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.5276329509906152, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 6337.8, "avg_prompt_tokens": 2003.1, "avg_response_tokens": 1120.2, "avg_reasoning_tokens": null, "cost_usd": 1.261342, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.506778, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 5035.6, "avg_prompt_tokens": 2025.6, "avg_response_tokens": 662.3, "avg_reasoning_tokens": null, "cost_usd": 0.417257, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.4744525547445255, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.4608967674661105, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 1126.7, "avg_prompt_tokens": 2210.4, "avg_response_tokens": 37.8, "avg_reasoning_tokens": null, "cost_usd": 0.087624, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.45776850886339937, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 7293, "avg_prompt_tokens": 2500.7, "avg_response_tokens": 602.5, "avg_reasoning_tokens": null, "cost_usd": 0.255956, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "law", "metric": "exact_match,custom-extract", "score": 0.41605839416058393, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 3293.9, "avg_prompt_tokens": 1988.2, "avg_response_tokens": 1073.7, "avg_reasoning_tokens": null, "cost_usd": 0.601918, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9459659511472984, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 2378.4, "avg_prompt_tokens": 2027.8, "avg_response_tokens": 409.6, "avg_reasoning_tokens": null, "cost_usd": 9.089742, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9459659511472984, "items_effective": 1351, "items_total": 1351, "reasoning_off": false, "median_latency_ms": 28795.8, "avg_prompt_tokens": 1928.6, "avg_response_tokens": 161.4, "avg_reasoning_tokens": 129.9, "cost_usd": 24.83407, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9407846039970392, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 3385.2, "avg_prompt_tokens": 2027.8, "avg_response_tokens": 386.3, "avg_reasoning_tokens": null, "cost_usd": 2.935318, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9385640266469282, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 1954.9, "avg_prompt_tokens": 2027.8, "avg_response_tokens": 331, "avg_reasoning_tokens": null, "cost_usd": 7.463002, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9348630643967432, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 67506, "avg_prompt_tokens": 2065.8, "avg_response_tokens": 521.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.926720947446336, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 4916.5, "avg_prompt_tokens": 2040.7, "avg_response_tokens": 898.6, "avg_reasoning_tokens": null, "cost_usd": 1.485059, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.920059, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 5175.1, "avg_prompt_tokens": 2065.6, "avg_response_tokens": 644.2, "avg_reasoning_tokens": null, "cost_usd": 0.559888, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9089563286454478, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 180.3, "avg_response_tokens": 250.2, "avg_reasoning_tokens": null, "cost_usd": 0.399996, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9045151739452256, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 2894.1, "avg_prompt_tokens": 2027.8, "avg_response_tokens": 898.8, "avg_reasoning_tokens": null, "cost_usd": 3.847715, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.9022945965951148, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 1332.8, "avg_prompt_tokens": 2028.9, "avg_response_tokens": 363.4, "avg_reasoning_tokens": null, "cost_usd": 1.416521, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.8985936343449297, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 1816.8, "avg_prompt_tokens": 2027.8, "avg_response_tokens": 426.7, "avg_reasoning_tokens": null, "cost_usd": 2.263057, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.8867505551443375, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 2241.4, "avg_prompt_tokens": 1966.8, "avg_response_tokens": 250.6, "avg_reasoning_tokens": null, "cost_usd": 0.134225, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.8675055514433753, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 3570.5, "avg_prompt_tokens": 4571.2, "avg_response_tokens": 452.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.8645447816432272, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 6135.9, "avg_prompt_tokens": 2116.8, "avg_response_tokens": 582.2, "avg_reasoning_tokens": null, "cost_usd": 0.314778, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "math", "metric": "exact_match,custom-extract", "score": 0.8282753515914137, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 2452.4, "avg_prompt_tokens": 2028.9, "avg_response_tokens": 1495.2, "avg_reasoning_tokens": null, "cost_usd": 1.066908, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_biology", "metric": "exact_match,custom-extract", "score": 0.8702928870292888, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 78951, "avg_prompt_tokens": 1875.7, "avg_response_tokens": 404.1, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_biology", "metric": "exact_match,custom-extract", "score": 0.8563458856345886, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 31594.1, "avg_prompt_tokens": 1863, "avg_response_tokens": 521.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_biology", "metric": "exact_match,custom-extract", "score": 0.8131101813110181, "items_effective": 717, "items_total": 717, "reasoning_off": true, "median_latency_ms": 22882.3, "avg_prompt_tokens": 2125.7, "avg_response_tokens": 290.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_business", "metric": "exact_match,custom-extract", "score": 0.7984790874524715, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 98875.7, "avg_prompt_tokens": 1434.1, "avg_response_tokens": 477.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_business", "metric": "exact_match,custom-extract", "score": 0.7807351077313055, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 30317.3, "avg_prompt_tokens": 1408.4, "avg_response_tokens": 1064.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_business", "metric": "exact_match,custom-extract", "score": 0.7034220532319392, "items_effective": 789, "items_total": 789, "reasoning_off": true, "median_latency_ms": 39749, "avg_prompt_tokens": 1602.1, "avg_response_tokens": 529.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_chemistry", "metric": "exact_match,custom-extract", "score": 0.8498233215547704, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 51969.5, "avg_prompt_tokens": 1865.8, "avg_response_tokens": 1190.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_chemistry", "metric": "exact_match,custom-extract", "score": 0.7985865724381626, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 114383.6, "avg_prompt_tokens": 1842, "avg_response_tokens": 690.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_chemistry", "metric": "exact_match,custom-extract", "score": 0.7217314487632509, "items_effective": 1132, "items_total": 1132, "reasoning_off": true, "median_latency_ms": 58565.9, "avg_prompt_tokens": 2006.4, "avg_response_tokens": 628.2, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_computer_science", "metric": "exact_match,custom-extract", "score": 0.7926829268292683, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 90266.6, "avg_prompt_tokens": 1895.9, "avg_response_tokens": 465.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_computer_science", "metric": "exact_match,custom-extract", "score": 0.7804878048780488, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 39925.7, "avg_prompt_tokens": 1879.6, "avg_response_tokens": 904.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_computer_science", "metric": "exact_match,custom-extract", "score": 0.7146341463414634, "items_effective": 410, "items_total": 410, "reasoning_off": true, "median_latency_ms": 24521.7, "avg_prompt_tokens": 2025.6, "avg_response_tokens": 380.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_economics", "metric": "exact_match,custom-extract", "score": 0.7890995260663507, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 29860.7, "avg_prompt_tokens": 1592.9, "avg_response_tokens": 579.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_economics", "metric": "exact_match,custom-extract", "score": 0.7867298578199052, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 94068.6, "avg_prompt_tokens": 1619.7, "avg_response_tokens": 423, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_economics", "metric": "exact_match,custom-extract", "score": 0.7109004739336493, "items_effective": 844, "items_total": 844, "reasoning_off": true, "median_latency_ms": 16723.5, "avg_prompt_tokens": 1774.9, "avg_response_tokens": 244, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_engineering", "metric": "exact_match,custom-extract", "score": 0.6656346749226006, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 103907.2, "avg_prompt_tokens": 1570.2, "avg_response_tokens": 2499.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_engineering", "metric": "exact_match,custom-extract", "score": 0.651186790505676, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 141606.6, "avg_prompt_tokens": 1587.3, "avg_response_tokens": 795.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_engineering", "metric": "exact_match,custom-extract", "score": 0.5593395252837977, "items_effective": 969, "items_total": 969, "reasoning_off": true, "median_latency_ms": 148694.9, "avg_prompt_tokens": 1669, "avg_response_tokens": 907, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_health", "metric": "exact_match,custom-extract", "score": 0.6593886462882096, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 29283.4, "avg_prompt_tokens": 1350.9, "avg_response_tokens": 581.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_health", "metric": "exact_match,custom-extract", "score": 0.6448326055312955, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 76108.8, "avg_prompt_tokens": 1356.1, "avg_response_tokens": 339.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_health", "metric": "exact_match,custom-extract", "score": 0.5676855895196506, "items_effective": 687, "items_total": 687, "reasoning_off": true, "median_latency_ms": 15030.6, "avg_prompt_tokens": 1510.5, "avg_response_tokens": 199, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_history", "metric": "exact_match,custom-extract", "score": 0.6036745406824147, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 49384.8, "avg_prompt_tokens": 2797.8, "avg_response_tokens": 605.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_history", "metric": "exact_match,custom-extract", "score": 0.5721784776902887, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 63813.7, "avg_prompt_tokens": 2811.4, "avg_response_tokens": 314.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_history", "metric": "exact_match,custom-extract", "score": 0.4881889763779528, "items_effective": 381, "items_total": 381, "reasoning_off": true, "median_latency_ms": 21077.5, "avg_prompt_tokens": 3292.8, "avg_response_tokens": 166.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_law", "metric": "exact_match,custom-extract", "score": 0.42752867570385816, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 88485, "avg_prompt_tokens": 2028.2, "avg_response_tokens": 473.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_law", "metric": "exact_match,custom-extract", "score": 0.3670490093847758, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 50851, "avg_prompt_tokens": 2007.1, "avg_response_tokens": 884.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_law", "metric": "exact_match,custom-extract", "score": 0.27424400417101147, "items_effective": 959, "items_total": 959, "reasoning_off": true, "median_latency_ms": 30276.1, "avg_prompt_tokens": 2407.5, "avg_response_tokens": 232, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_math", "metric": "exact_match,custom-extract", "score": 0.9015544041450776, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 49562.6, "avg_prompt_tokens": 2044.7, "avg_response_tokens": 945.1, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_math", "metric": "exact_match,custom-extract", "score": 0.9008142116950408, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 105546.9, "avg_prompt_tokens": 2068.8, "avg_response_tokens": 556.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_math", "metric": "exact_match,custom-extract", "score": 0.8216136195410807, "items_effective": 1351, "items_total": 1351, "reasoning_off": true, "median_latency_ms": 50411.3, "avg_prompt_tokens": 2162.8, "avg_response_tokens": 508.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_other", "metric": "exact_match,custom-extract", "score": 0.6158008658008658, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 21318.2, "avg_prompt_tokens": 1229.7, "avg_response_tokens": 423.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_other", "metric": "exact_match,custom-extract", "score": 0.6071428571428571, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 52643.3, "avg_prompt_tokens": 1224.6, "avg_response_tokens": 243, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_other", "metric": "exact_match,custom-extract", "score": 0.5205627705627706, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 11771.2, "avg_prompt_tokens": 1358.2, "avg_response_tokens": 195.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_philosophy", "metric": "exact_match,custom-extract", "score": 0.6032064128256514, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 76872.2, "avg_prompt_tokens": 1323.9, "avg_response_tokens": 367.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_philosophy", "metric": "exact_match,custom-extract", "score": 0.5811623246492986, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 23254.7, "avg_prompt_tokens": 1303.6, "avg_response_tokens": 499.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_philosophy", "metric": "exact_match,custom-extract", "score": 0.49298597194388777, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 11210.5, "avg_prompt_tokens": 1431.5, "avg_response_tokens": 222.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_physics", "metric": "exact_match,custom-extract", "score": 0.8468052347959969, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 43850.6, "avg_prompt_tokens": 1573.3, "avg_response_tokens": 1086.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_physics", "metric": "exact_match,custom-extract", "score": 0.8190916089299461, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 109709.6, "avg_prompt_tokens": 1578.4, "avg_response_tokens": 616.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_physics", "metric": "exact_match,custom-extract", "score": 0.7305619707467282, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 75893.1, "avg_prompt_tokens": 1737.7, "avg_response_tokens": 590.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_psychology", "metric": "exact_match,custom-extract", "score": 0.7631578947368421, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 76742.2, "avg_prompt_tokens": 1542.2, "avg_response_tokens": 333.2, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-08-57__openai__gemma4-12b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_psychology", "metric": "exact_match,custom-extract", "score": 0.7418546365914787, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 25760.3, "avg_prompt_tokens": 1513.5, "avg_response_tokens": 372, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-43-11__openai__qwen35-9b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "nothink_psychology", "metric": "exact_match,custom-extract", "score": 0.6604010025062657, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 14032.9, "avg_prompt_tokens": 1802.4, "avg_response_tokens": 149.1, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-03-02__openai__qwen3-14b-bf16__mmlu_prox_de_nothink" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.8430735930735931, "items_effective": 924, "items_total": 924, "reasoning_off": false, "median_latency_ms": 25743.8, "avg_prompt_tokens": 1136.7, "avg_response_tokens": 76.6, "avg_reasoning_tokens": 95.6, "cost_usd": 10.02433, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.8257575757575758, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 2226.4, "avg_prompt_tokens": 1183.6, "avg_response_tokens": 135.4, "avg_reasoning_tokens": null, "cost_usd": 0.92226, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.8235930735930735, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 1656.8, "avg_prompt_tokens": 1183.6, "avg_response_tokens": 159.6, "avg_reasoning_tokens": null, "cost_usd": 2.967794, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.814935064935065, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 1302.1, "avg_prompt_tokens": 1183.6, "avg_response_tokens": 120.9, "avg_reasoning_tokens": null, "cost_usd": 2.478624, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.7727272727272727, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 216.2, "avg_response_tokens": 104.7, "avg_reasoning_tokens": null, "cost_usd": 0.171091, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.7673160173160173, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 854.3, "avg_prompt_tokens": 1184.6, "avg_response_tokens": 139.8, "avg_reasoning_tokens": null, "cost_usd": 0.464846, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.7532467532467533, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 1389.3, "avg_prompt_tokens": 1183.6, "avg_response_tokens": 128.2, "avg_reasoning_tokens": null, "cost_usd": 0.624328, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.737012987012987, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 38145.5, "avg_prompt_tokens": 1221.6, "avg_response_tokens": 310.2, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.7348484848484849, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 1055.2, "avg_prompt_tokens": 1183.6, "avg_response_tokens": 195.1, "avg_reasoning_tokens": null, "cost_usd": 0.778861, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.7207792207792207, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 4370.5, "avg_prompt_tokens": 1400.8, "avg_response_tokens": 260, "avg_reasoning_tokens": null, "cost_usd": 0.118668, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.7077922077922078, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 2503, "avg_prompt_tokens": 1225.7, "avg_response_tokens": 477.6, "avg_reasoning_tokens": null, "cost_usd": 0.60499, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.6948051948051948, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 3554.5, "avg_prompt_tokens": 3656.3, "avg_response_tokens": 332.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.6937229437229437, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 1431.7, "avg_prompt_tokens": 1235.9, "avg_response_tokens": 62.1, "avg_reasoning_tokens": null, "cost_usd": 0.04591, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.675325, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 4646.4, "avg_prompt_tokens": 1221.8, "avg_response_tokens": 455.8, "avg_reasoning_tokens": null, "cost_usd": 0.25916, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "other", "metric": "exact_match,custom-extract", "score": 0.6493506493506493, "items_effective": 924, "items_total": 924, "reasoning_off": true, "median_latency_ms": 1092, "avg_prompt_tokens": 1184.6, "avg_response_tokens": 480.7, "avg_reasoning_tokens": null, "cost_usd": 0.287135, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.8517034068136272, "items_effective": 499, "items_total": 499, "reasoning_off": false, "median_latency_ms": 26588.3, "avg_prompt_tokens": 1235.3, "avg_response_tokens": 77.2, "avg_reasoning_tokens": 119.7, "cost_usd": 6.029605, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.8316633266533067, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 1804.2, "avg_prompt_tokens": 1282.9, "avg_response_tokens": 210.1, "avg_reasoning_tokens": null, "cost_usd": 1.903938, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.8236472945891784, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 2679, "avg_prompt_tokens": 1282.9, "avg_response_tokens": 206.7, "avg_reasoning_tokens": null, "cost_usd": 0.629528, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.8236472945891784, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 1511.7, "avg_prompt_tokens": 1282.9, "avg_response_tokens": 159.8, "avg_reasoning_tokens": null, "cost_usd": 1.558436, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.751503006012024, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 1030.9, "avg_prompt_tokens": 1284.1, "avg_response_tokens": 197.6, "avg_reasoning_tokens": null, "cost_usd": 0.305041, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.7434869739478958, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 51663.9, "avg_prompt_tokens": 1320.9, "avg_response_tokens": 421.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.7374749498997996, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 217.6, "avg_response_tokens": 128, "avg_reasoning_tokens": null, "cost_usd": 0.102813, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.7294589178356713, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 1482.9, "avg_prompt_tokens": 1282.9, "avg_response_tokens": 197.4, "avg_reasoning_tokens": null, "cost_usd": 0.438298, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.7194388777555111, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 2013, "avg_prompt_tokens": 1327.5, "avg_response_tokens": 126.7, "avg_reasoning_tokens": null, "cost_usd": 0.033742, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.7094188376753507, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 2041.6, "avg_prompt_tokens": 1282.9, "avg_response_tokens": 482.5, "avg_reasoning_tokens": null, "cost_usd": 0.793988, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.6953907815631263, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 3444.5, "avg_prompt_tokens": 1299.6, "avg_response_tokens": 684.3, "avg_reasoning_tokens": null, "cost_usd": 0.38627, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.691383, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 12187.6, "avg_prompt_tokens": 1321.2, "avg_response_tokens": 571.3, "avg_reasoning_tokens": null, "cost_usd": 0.16674, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.6913827655310621, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 6057.2, "avg_prompt_tokens": 1476.4, "avg_response_tokens": 461.4, "avg_reasoning_tokens": null, "cost_usd": 0.093749, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.6793587174348698, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 3987.4, "avg_prompt_tokens": 3757.3, "avg_response_tokens": 403.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "philosophy", "metric": "exact_match,custom-extract", "score": 0.6092184368737475, "items_effective": 499, "items_total": 499, "reasoning_off": true, "median_latency_ms": 2453.6, "avg_prompt_tokens": 1283.9, "avg_response_tokens": 857.4, "avg_reasoning_tokens": null, "cost_usd": 0.235201, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.907621247113164, "items_effective": 1299, "items_total": 1299, "reasoning_off": false, "median_latency_ms": 34217.8, "avg_prompt_tokens": 1455.9, "avg_response_tokens": 202.8, "avg_reasoning_tokens": 141.8, "cost_usd": 22.88582, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.9037721324095458, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 2674.4, "avg_prompt_tokens": 1537.4, "avg_response_tokens": 458.9, "avg_reasoning_tokens": null, "cost_usd": 8.360724, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.9037721324095458, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 1903.9, "avg_prompt_tokens": 1537.4, "avg_response_tokens": 323.7, "avg_reasoning_tokens": null, "cost_usd": 6.149397, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.9030023094688222, "items_effective": 1299, "items_total": 1299, "reasoning_off": false, "median_latency_ms": 6071.1, "avg_prompt_tokens": 1537.4, "avg_response_tokens": 371.4, "avg_reasoning_tokens": 0.5, "cost_usd": 2.447821, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.8899153194765204, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 78274.9, "avg_prompt_tokens": 1575.4, "avg_response_tokens": 562.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.8799076212471132, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 4620.7, "avg_prompt_tokens": 1569.3, "avg_response_tokens": 1097.2, "avg_reasoning_tokens": null, "cost_usd": 1.594115, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.876058506543495, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 1399.4, "avg_prompt_tokens": 1538.3, "avg_response_tokens": 369.6, "avg_reasoning_tokens": null, "cost_usd": 1.214988, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.8683602771362586, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 268.5, "avg_response_tokens": 281.9, "avg_reasoning_tokens": null, "cost_usd": 0.470383, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.860662, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 7130, "avg_prompt_tokens": 1575.8, "avg_response_tokens": 784.1, "avg_reasoning_tokens": null, "cost_usd": 0.568346, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.859122401847575, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 1794.3, "avg_prompt_tokens": 1537.4, "avg_response_tokens": 419.9, "avg_reasoning_tokens": null, "cost_usd": 1.96275, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.8552732871439569, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 3066.8, "avg_prompt_tokens": 1537.4, "avg_response_tokens": 967.4, "avg_reasoning_tokens": null, "cost_usd": 3.740674, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.8468052347959969, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 2586.8, "avg_prompt_tokens": 1543.8, "avg_response_tokens": 281.2, "avg_reasoning_tokens": null, "cost_usd": 0.155124, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.812933025404157, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 3771.3, "avg_prompt_tokens": 3997.2, "avg_response_tokens": 447.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.7659738260200154, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 3059.3, "avg_prompt_tokens": 1538.2, "avg_response_tokens": 2208.9, "avg_reasoning_tokens": null, "cost_usd": 1.343395, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "physics", "metric": "exact_match,custom-extract", "score": 0.6474210931485759, "items_effective": 1299, "items_total": 1299, "reasoning_off": true, "median_latency_ms": 6407.2, "avg_prompt_tokens": 1710.4, "avg_response_tokens": 624.9, "avg_reasoning_tokens": null, "cost_usd": 0.301034, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8784461152882206, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 1891, "avg_prompt_tokens": 1501.2, "avg_response_tokens": 191.7, "avg_reasoning_tokens": null, "cost_usd": 3.173948, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-24__vertex_ai__gemini-3.5-flash" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8784461152882206, "items_effective": 798, "items_total": 798, "reasoning_off": false, "median_latency_ms": 3733.9, "avg_prompt_tokens": 1501.2, "avg_response_tokens": 166.1, "avg_reasoning_tokens": 0.4, "cost_usd": 0.99747, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T16-07-36__vertex_ai__gemini-3-flash-preview__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8734335839598998, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 1503.1, "avg_prompt_tokens": 1501.2, "avg_response_tokens": 161.1, "avg_reasoning_tokens": null, "cost_usd": 2.760963, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-23T00-09-20__vertex_ai__gemini-3.6-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8634085213032582, "items_effective": 798, "items_total": 798, "reasoning_off": false, "median_latency_ms": 21775.6, "avg_prompt_tokens": 1460.4, "avg_response_tokens": 87.2, "avg_reasoning_tokens": 70.2, "cost_usd": 9.596765, "quantization": "provider-internal", "run_date": "2026-07-01", "run_id": "2026-07-01T09-02-53__openai__codex__gpt-5.5-low__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8395989974937343, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 1398.1, "avg_prompt_tokens": 1501.2, "avg_response_tokens": 117.5, "avg_reasoning_tokens": null, "cost_usd": 0.593893, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T23-51-12__vertex_ai__gemini-3.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8358395989974937, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 43857.9, "avg_prompt_tokens": 1539.2, "avg_response_tokens": 359.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T15-10-23__openai__gemma-4-31b-it-bf16__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8333333333333334, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 949, "avg_prompt_tokens": 1502.3, "avg_response_tokens": 156.8, "avg_reasoning_tokens": null, "cost_usd": 0.484372, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-05-15__gemini__gemini-3.1-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8333333333333334, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": 276.8, "avg_response_tokens": 114.6, "avg_reasoning_tokens": null, "cost_usd": 0.175639, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T13-49-10__openrouter__deepseek__deepseek-v4-pro__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8258145363408521, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 1415, "avg_prompt_tokens": 1501.2, "avg_response_tokens": 241.2, "avg_reasoning_tokens": null, "cost_usd": 0.840598, "quantization": "provider-internal", "run_date": "2026-05-26", "run_id": "2026-05-26T15-52-44__vertex_ai__gemini-2.5-flash" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8082706766917294, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 1597.2, "avg_prompt_tokens": 1674.1, "avg_response_tokens": 67.4, "avg_reasoning_tokens": null, "cost_usd": 0.047695, "quantization": "provider-internal", "run_date": "2026-06-16", "run_id": "2026-06-16T12-31-34__openrouter__deepseek__deepseek-v4-flash__mmlu_prox_de" }, { "kind": "subject", "model_slug": "tencent/hy3-preview", "model_display": "Tencent HY3-Preview", "model_vendor": "Tencent", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.8045112781954887, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 4731.3, "avg_prompt_tokens": 1903.4, "avg_response_tokens": 284.2, "avg_reasoning_tokens": null, "cost_usd": 0.124799, "quantization": "provider-internal", "run_date": "2026-05-27", "run_id": "2026-05-27T23-43-24__openrouter__tencent__hy3-preview" }, { "kind": "subject", "model_slug": "anthropic/claude-haiku-4.5", "model_display": "Claude Haiku 4.5", "model_vendor": "Anthropic", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.7894736842105263, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 3910.2, "avg_prompt_tokens": 4252.8, "avg_response_tokens": 382.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "", "run_date": "2026-06-03", "run_id": "2026-06-03T15-52-51__openai__cc__claude-haiku-4-5-20251001__mmlu_prox_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.6-35b-a3b", "model_display": "Qwen3.6 35B-A3B", "model_vendor": "Alibaba", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.7869674185463659, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 3593, "avg_prompt_tokens": 1509.5, "avg_response_tokens": 585.7, "avg_reasoning_tokens": null, "cost_usd": 0.564006, "quantization": "fp8", "run_date": "2026-05-31", "run_id": "2026-05-31T12-32-33__openrouter__qwen__qwen3.6-35b-a3b" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.785714, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 5830.3, "avg_prompt_tokens": 1539.4, "avg_response_tokens": 501.7, "avg_reasoning_tokens": null, "cost_usd": 0.257794, "quantization": "bf16", "run_date": "2026-06-05", "run_id": "2026-06-05T10-17-05__openrouter__google__gemma-4-26b-a4b-it__mmlu_prox_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash-lite", "model_display": "Gemini 2.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "mmlu_prox_de", "benchmark_display": "MMLU-ProX (DE)", "subject": "psychology", "metric": "exact_match,custom-extract", "score": 0.7506265664160401, "items_effective": 798, "items_total": 798, "reasoning_off": true, "median_latency_ms": 2138.5, "avg_prompt_tokens": 1502.2, "avg_response_tokens": 681.9, "avg_reasoning_tokens": null, "cost_usd": 0.337541, "quantization": "provider-internal", "run_date": "2026-06-03", "run_id": "2026-06-03T16-49-00__gemini__gemini-2.5-flash-lite__mmlu_prox_de" }, { "kind": "subject", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.928, "items_effective": 250, "items_total": 250, "reasoning_off": false, "median_latency_ms": 19892.5, "avg_prompt_tokens": 3587.7, "avg_response_tokens": 1263.6, "avg_reasoning_tokens": null, "cost_usd": 24.76462, "quantization": "provider-internal", "run_date": "2026-07-05", "run_id": "2026-07-05T22-33-14__openai__cc__claude-fable-5__musr_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.9, "items_effective": 250, "items_total": 250, "reasoning_off": false, "median_latency_ms": 13230.6, "avg_prompt_tokens": 1768.8, "avg_response_tokens": 449.3, "avg_reasoning_tokens": 131.6, "cost_usd": 6.56802, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-13__openai__codex__gpt-5.5-low__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.892, "items_effective": 250, "items_total": 250, "reasoning_off": false, "median_latency_ms": 11494.9, "avg_prompt_tokens": 1795.5, "avg_response_tokens": 484.8, "avg_reasoning_tokens": 792.8, "cost_usd": 4.730548, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-30-10__vertex_ai__gemini-3.1-pro-preview__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.876, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 5466.7, "avg_prompt_tokens": 1795.5, "avg_response_tokens": 794.5, "avg_reasoning_tokens": null, "cost_usd": 0.630645, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-10-07__vertex_ai__gemini-2.5-flash__musr_de" }, { "kind": "subject", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.876, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 13196.8, "avg_prompt_tokens": 3618.7, "avg_response_tokens": 991.9, "avg_reasoning_tokens": null, "cost_usd": 10.72261, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-02__openai__cc__claude-opus-4-8__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.864, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 21084.9, "avg_prompt_tokens": 1808.5, "avg_response_tokens": 555.4, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-02-37__openai__gemma-4-31b-it-bf16__musr_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.864, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 12145.4, "avg_prompt_tokens": 2009.9, "avg_response_tokens": 584.2, "avg_reasoning_tokens": null, "cost_usd": 0.340493, "quantization": "provider-internal", "run_date": "2026-06-23", "run_id": "2026-06-23T13-55-55__openrouter__deepseek__deepseek-v4-pro__musr_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.856, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 10792.4, "avg_prompt_tokens": 2012.4, "avg_response_tokens": 762.7, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "native", "run_date": "2026-06-10", "run_id": "2026-06-10T12-44-30__openrouter__deepseek__deepseek-v4-flash__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.856, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 207721.4, "avg_prompt_tokens": 1808.5, "avg_response_tokens": 627.2, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-52-24__openai__gemma4-12b-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.852, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.852, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 2774.5, "avg_prompt_tokens": 1795.5, "avg_response_tokens": 577, "avg_reasoning_tokens": null, "cost_usd": 0.49528, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-57-34__vertex_ai__gemini-3.5-flash-lite__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.848, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 2986.3, "avg_prompt_tokens": 1798.3, "avg_response_tokens": 543, "avg_reasoning_tokens": null, "cost_usd": 0.331199, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-44-23__vertex_ai__gemini-3.1-flash-lite__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.84, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 5360.4, "avg_prompt_tokens": 1795.5, "avg_response_tokens": 790.8, "avg_reasoning_tokens": null, "cost_usd": 2.452638, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-54-48__vertex_ai__gemini-3.5-flash__musr_de" }, { "kind": "subject", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.8, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.796, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 3624, "avg_prompt_tokens": 1795.5, "avg_response_tokens": 620.8, "avg_reasoning_tokens": null, "cost_usd": 1.837265, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-59-41__vertex_ai__gemini-3.6-flash__musr_de" }, { "kind": "subject", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.788, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.788, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 5376, "avg_prompt_tokens": 1795.5, "avg_response_tokens": 615.8, "avg_reasoning_tokens": null, "cost_usd": 0.686317, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__musr_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.776, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 59604.7, "avg_prompt_tokens": 1805.1, "avg_response_tokens": 1200.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-33-49__openai__qwen35-9b-bf16__musr_de" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "murder_mystery", "metric": "acc,none", "score": 0.768, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 257890.6, "avg_prompt_tokens": 2077.4, "avg_response_tokens": 1037.5, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-34-46__openai__qwen3-14b-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.8125, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 3680.5, "avg_prompt_tokens": 1527.5, "avg_response_tokens": 484.3, "avg_reasoning_tokens": null, "cost_usd": 0.425625, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-54-48__vertex_ai__gemini-3.5-flash__musr_de" }, { "kind": "subject", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.8125, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.8125, "items_effective": 64, "items_total": 64, "reasoning_off": false, "median_latency_ms": 15425.2, "avg_prompt_tokens": 3140.1, "avg_response_tokens": 931.4, "avg_reasoning_tokens": null, "cost_usd": 4.99006, "quantization": "provider-internal", "run_date": "2026-07-05", "run_id": "2026-07-05T22-33-14__openai__cc__claude-fable-5__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.796875, "items_effective": 64, "items_total": 64, "reasoning_off": false, "median_latency_ms": 8594, "avg_prompt_tokens": 1527.5, "avg_response_tokens": 340.4, "avg_reasoning_tokens": 585.7, "cost_usd": 0.906764, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-30-10__vertex_ai__gemini-3.1-pro-preview__musr_de" }, { "kind": "subject", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.796875, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 9041.1, "avg_prompt_tokens": 3171.1, "avg_response_tokens": 677.3, "avg_reasoning_tokens": null, "cost_usd": 2.098375, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-02__openai__cc__claude-opus-4-8__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.796875, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 57316.7, "avg_prompt_tokens": 1540.5, "avg_response_tokens": 498.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-52-24__openai__gemma4-12b-bf16__musr_de" }, { "kind": "subject", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.765625, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.765625, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 16174.8, "avg_prompt_tokens": 1540.5, "avg_response_tokens": 438.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-02-37__openai__gemma-4-31b-it-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.765625, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 4142, "avg_prompt_tokens": 1527.5, "avg_response_tokens": 430.2, "avg_reasoning_tokens": null, "cost_usd": 0.131471, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.75, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 2250.3, "avg_prompt_tokens": 1540.2, "avg_response_tokens": 386.6, "avg_reasoning_tokens": null, "cost_usd": 0.073332, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-44-23__vertex_ai__gemini-3.1-flash-lite__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.75, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 3708.2, "avg_prompt_tokens": 1527.5, "avg_response_tokens": 527.2, "avg_reasoning_tokens": null, "cost_usd": 0.113689, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-10-07__vertex_ai__gemini-2.5-flash__musr_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.75, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 6841.2, "avg_prompt_tokens": 1763.5, "avg_response_tokens": 488, "avg_reasoning_tokens": null, "cost_usd": 0.024546, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "2026-06-10T11-50-02__openrouter__deepseek__deepseek-v4-flash__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.75, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 2291, "avg_prompt_tokens": 1527.5, "avg_response_tokens": 467, "avg_reasoning_tokens": null, "cost_usd": 0.104054, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-57-34__vertex_ai__gemini-3.5-flash-lite__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.734375, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.734375, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 23039.1, "avg_prompt_tokens": 1535.9, "avg_response_tokens": 897.2, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-33-49__openai__qwen35-9b-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.734375, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 2331.8, "avg_prompt_tokens": 1527.5, "avg_response_tokens": 388.8, "avg_reasoning_tokens": null, "cost_usd": 0.333251, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-59-41__vertex_ai__gemini-3.6-flash__musr_de" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.71875, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 56089.1, "avg_prompt_tokens": 1829.2, "avg_response_tokens": 711.8, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-34-46__openai__qwen3-14b-bf16__musr_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.703125, "items_effective": 64, "items_total": 64, "reasoning_off": true, "median_latency_ms": 10004.6, "avg_prompt_tokens": 1757.2, "avg_response_tokens": 518.3, "avg_reasoning_tokens": null, "cost_usd": 0.072769, "quantization": "provider-internal", "run_date": "2026-06-23", "run_id": "2026-06-23T13-55-55__openrouter__deepseek__deepseek-v4-pro__musr_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "object_placements", "metric": "acc,none", "score": 0.6875, "items_effective": 64, "items_total": 64, "reasoning_off": false, "median_latency_ms": 9666.7, "avg_prompt_tokens": 1516.6, "avg_response_tokens": 223.9, "avg_reasoning_tokens": 245.3, "cost_usd": 1.386085, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-13__openai__codex__gpt-5.5-low__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-pro-preview", "model_display": "Gemini 3.1 Pro", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.892, "items_effective": 250, "items_total": 250, "reasoning_off": false, "median_latency_ms": 13550.2, "avg_prompt_tokens": 1118.4, "avg_response_tokens": 580.4, "avg_reasoning_tokens": 852.4, "cost_usd": 4.857542, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-30-10__vertex_ai__gemini-3.1-pro-preview__musr_de" }, { "kind": "subject", "model_slug": "anthropic/claude-fable-5", "model_display": "Fable 5", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.884, "items_effective": 250, "items_total": 250, "reasoning_off": false, "median_latency_ms": 21197.1, "avg_prompt_tokens": 2396.3, "avg_response_tokens": 1516.9, "avg_reasoning_tokens": null, "cost_usd": 24.95208, "quantization": "provider-internal", "run_date": "2026-07-05", "run_id": "2026-07-05T22-33-14__openai__cc__claude-fable-5__musr_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-pro", "model_display": "DeepSeek V4 Pro", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.88, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 14131.5, "avg_prompt_tokens": 1280.2, "avg_response_tokens": 906.7, "avg_reasoning_tokens": null, "cost_usd": 0.331524, "quantization": "provider-internal", "run_date": "2026-06-23", "run_id": "2026-06-23T13-55-55__openrouter__deepseek__deepseek-v4-pro__musr_de" }, { "kind": "subject", "model_slug": "openai/gpt-5.5", "model_display": "GPT-5.5", "model_vendor": "OpenAI", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.872, "items_effective": 250, "items_total": 250, "reasoning_off": false, "median_latency_ms": 14609.9, "avg_prompt_tokens": 1112.9, "avg_response_tokens": 429.2, "avg_reasoning_tokens": 262.2, "cost_usd": 6.57677, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-13__openai__codex__gpt-5.5-low__musr_de" }, { "kind": "subject", "model_slug": "anthropic/claude-opus-4.8", "model_display": "Opus 4.8", "model_vendor": "Anthropic", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.868, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 15512.3, "avg_prompt_tokens": 2427.3, "avg_response_tokens": 1261.4, "avg_reasoning_tokens": null, "cost_usd": 10.91784, "quantization": "provider-internal", "run_date": "2026-06-13", "run_id": "2026-06-13T18-12-02__openai__cc__claude-opus-4-8__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.1-flash-lite", "model_display": "Gemini 3.1 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.864, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 3039.8, "avg_prompt_tokens": 1115.2, "avg_response_tokens": 646.8, "avg_reasoning_tokens": null, "cost_usd": 0.327226, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-44-23__vertex_ai__gemini-3.1-flash-lite__musr_de" }, { "kind": "subject", "model_slug": "qwen/qwen3.5-9b", "model_display": "Qwen3.5 9B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.86, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 76592.5, "avg_prompt_tokens": 1116.3, "avg_response_tokens": 2246.6, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-33-49__openai__qwen35-9b-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash", "model_display": "Gemini 3.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.852, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 6456.1, "avg_prompt_tokens": 1118.4, "avg_response_tokens": 1003.2, "avg_reasoning_tokens": null, "cost_usd": 2.676665, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T17-54-48__vertex_ai__gemini-3.5-flash__musr_de" }, { "kind": "subject", "model_slug": "deepseek/deepseek-v4-flash", "model_display": "DeepSeek V4 Flash", "model_vendor": "DeepSeek", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.848, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 11575.4, "avg_prompt_tokens": 1280.7, "avg_response_tokens": 916.1, "avg_reasoning_tokens": null, "cost_usd": 0.108278, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "2026-06-10T11-50-02__openrouter__deepseek__deepseek-v4-flash__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-26b-a4b-it", "model_display": "Gemma 4 26B A4B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.848, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "xiaomi/mimo-v2.5-pro", "model_display": "MiMo V2.5 Pro", "model_vendor": "Xiaomi", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.84, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "z-ai/glm-5.1", "model_display": "GLM-5.1", "model_vendor": "Z.ai", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.828, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": null, "avg_prompt_tokens": null, "avg_response_tokens": null, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "fp8", "run_date": "2026-06-10", "run_id": "musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.6-flash", "model_display": "Gemini 3.6 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.828, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 4499.8, "avg_prompt_tokens": 1118.4, "avg_response_tokens": 777.1, "avg_reasoning_tokens": null, "cost_usd": 1.876479, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-59-41__vertex_ai__gemini-3.6-flash__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-31b-it", "model_display": "Gemma 4 31B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.824, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 26748.2, "avg_prompt_tokens": 1131.4, "avg_response_tokens": 798.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-02-37__openai__gemma-4-31b-it-bf16__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-2.5-flash", "model_display": "Gemini 2.5 Flash", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.812, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 8411.5, "avg_prompt_tokens": 1118.4, "avg_response_tokens": 1501.2, "avg_reasoning_tokens": null, "cost_usd": 1.022155, "quantization": "provider-internal", "run_date": "2026-06-09", "run_id": "2026-06-09T18-10-07__vertex_ai__gemini-2.5-flash__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3-flash-preview", "model_display": "Gemini 3 Flash Preview", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.804, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 5535.2, "avg_prompt_tokens": 1118.4, "avg_response_tokens": 679.4, "avg_reasoning_tokens": null, "cost_usd": 0.649357, "quantization": "provider-internal", "run_date": "2026-06-25", "run_id": "2026-06-25T11-34-32__vertex_ai__gemini-3-flash-preview__musr_de" }, { "kind": "subject", "model_slug": "google/gemini-3.5-flash-lite", "model_display": "Gemini 3.5 Flash-Lite", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.8, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 3106.2, "avg_prompt_tokens": 1118.4, "avg_response_tokens": 768.3, "avg_reasoning_tokens": null, "cost_usd": 0.564048, "quantization": "provider-internal", "run_date": "2026-07-23", "run_id": "2026-07-22T22-57-34__vertex_ai__gemini-3.5-flash-lite__musr_de" }, { "kind": "subject", "model_slug": "google/gemma-4-12b-it", "model_display": "Gemma 4 12B", "model_vendor": "Google", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.78, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 197545.6, "avg_prompt_tokens": 1131.4, "avg_response_tokens": 840.9, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-52-24__openai__gemma4-12b-bf16__musr_de" }, { "kind": "subject", "model_slug": "qwen/qwen3-14b", "model_display": "Qwen3 14B", "model_vendor": "Alibaba", "benchmark_key": "musr_de", "benchmark_display": "MuSR (DE)", "subject": "team_allocation", "metric": "acc,none", "score": 0.62, "items_effective": 250, "items_total": 250, "reasoning_off": true, "median_latency_ms": 390182.1, "avg_prompt_tokens": 1340.6, "avg_response_tokens": 1660.3, "avg_reasoning_tokens": null, "cost_usd": null, "quantization": "bf16", "run_date": "2026-06-11", "run_id": "2026-06-11T14-34-46__openai__qwen3-14b-bf16__musr_de" } ] }