{ "name": "LLM Verifications AI model benchmark results", "publisher": "LLM Verifications", "editor": "LLM Verifications team", "data_as_of": "2026-10-27T17:31:31.658Z", "methodology": "https://www.llmverifications.com/methodology", "terms": "https://www.llmverifications.com/terms", "citation": "LLM Verifications team. LLM Verifications AI model benchmark results, data as of 2026-10-27. https://www.llmverifications.com/downloads", "suite": { "name": "Verification suite v2", "questions": 80, "sha256": "cd444ee142dc6b87e74967c9925ea9b7b171b90fe0e18b5a214c4587d964121a" }, "datasets": [ { "id": "supergpqa", "name": "SuperGPQA", "questions": 50, "license": "ODC-BY 1.0", "url": "https://huggingface.co/datasets/m-a-p/SuperGPQA", "paper": "https://arxiv.org/abs/2502.14739" }, { "id": "mmlu-pro", "name": "MMLU-Pro", "questions": 30, "license": "MIT", "url": "https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro", "paper": "https://arxiv.org/abs/2406.01574" } ], "speed_runs_per_model": 3, "models": [ { "model_id": "thinkingmachines/inkling", "name": "Inkling", "creator": "Thinking Machines", "url": "https://www.llmverifications.com/models/inkling", "open_weights": true, "reasoning": true, "score_pct": 66.2, "correct": 53, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001238, "cost_per_correct_usd": 0.00187, "output_tokens_per_s": 117.9, "time_to_first_token_s": 0.553, "end_to_end_s": 4.98, "output_tokens_per_question": 228, "context_window": 524288, "list_input_usd_per_m": 1, "list_output_usd_per_m": 4.05, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "27/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "21/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "stepfun/step-5-preview", "name": "Step 5 Preview", "creator": "StepFun", "url": "https://www.llmverifications.com/models/step-5-preview", "open_weights": true, "reasoning": true, "score_pct": 63.8, "correct": 51, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.004704, "cost_per_correct_usd": 0.007373, "output_tokens_per_s": 92.7, "time_to_first_token_s": 0.629, "end_to_end_s": 8.16, "output_tokens_per_question": 1630, "context_window": 1048576, "list_input_usd_per_m": 1, "list_output_usd_per_m": 2.7, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "26/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "24/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "mistralai/mistral-medium-3-5", "name": "Mistral Medium 3.5", "creator": "Mistral", "url": "https://www.llmverifications.com/models/mistral-medium-3-5", "open_weights": null, "reasoning": true, "score_pct": 28.8, "correct": 23, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.024982, "cost_per_correct_usd": 0.086744, "output_tokens_per_s": 145, "time_to_first_token_s": 0.458, "end_to_end_s": 5.36, "output_tokens_per_question": 3268, "context_window": 262144, "list_input_usd_per_m": 1.5, "list_output_usd_per_m": 7.5, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "5/50", "mmlu-pro_correct": "18/30", "subject_science_correct": "5/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "3/11", "subject_economics_correct": "6/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "qwen/qwen3.8-2.4t-a95b", "name": "Qwen3.8 2.4T A95B", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-8-2-4t-a95b", "open_weights": true, "reasoning": true, "score_pct": 57.5, "correct": 46, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.01518, "cost_per_correct_usd": 0.0264, "output_tokens_per_s": 44.3, "time_to_first_token_s": 3.143, "end_to_end_s": 14.79, "output_tokens_per_question": 2408, "context_window": 1048576, "list_input_usd_per_m": 2, "list_output_usd_per_m": 6, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "23/50", "mmlu-pro_correct": "23/30", "subject_science_correct": "21/30", "subject_engineering_correct": "5/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "moonshotai/kimi-k2.7-code", "name": "Kimi K2.7 Code", "creator": "MoonshotAI", "url": "https://www.llmverifications.com/models/kimi-k2-7-code", "open_weights": true, "reasoning": true, "score_pct": 65, "correct": 52, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.006006, "cost_per_correct_usd": 0.00924, "output_tokens_per_s": 136.4, "time_to_first_token_s": 0.362, "end_to_end_s": 5.26, "output_tokens_per_question": 1781, "context_window": 262144, "list_input_usd_per_m": 0.56, "list_output_usd_per_m": 3.1, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "25/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "22/30", "subject_engineering_correct": "7/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "7/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "cohere/command-a", "name": "Command A", "creator": "Cohere", "url": "https://www.llmverifications.com/models/command-a", "open_weights": true, "reasoning": false, "score_pct": 45, "correct": 36, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.00697, "cost_per_correct_usd": 0.01549, "output_tokens_per_s": 7.2, "time_to_first_token_s": 1.846, "end_to_end_s": 53.35, "output_tokens_per_question": 616, "context_window": 256000, "list_input_usd_per_m": 2.5, "list_output_usd_per_m": 10, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "12/50", "mmlu-pro_correct": "24/30", "subject_science_correct": "12/30", "subject_engineering_correct": "3/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "6/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "rekaai/reka-flash-3", "name": "Reka Flash 3", "creator": "rekaai", "url": "https://www.llmverifications.com/models/reka-flash-3", "open_weights": true, "reasoning": true, "score_pct": 1.2, "correct": 1, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000803, "cost_per_correct_usd": 0.066915, "output_tokens_per_s": 101.4, "time_to_first_token_s": 0.476, "end_to_end_s": 7.38, "output_tokens_per_question": 3855, "context_window": 65536, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.2, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "1/50", "mmlu-pro_correct": "0/30", "subject_science_correct": "0/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "1/11", "subject_economics_correct": "0/9", "subject_law_correct": "0/6", "subject_humanities_social_science_correct": "0/9" }, { "model_id": "meta-llama/llama-4-scout", "name": "Llama 4 Scout", "creator": "Meta", "url": "https://www.llmverifications.com/models/llama-4-scout", "open_weights": true, "reasoning": false, "score_pct": 41.2, "correct": 33, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000369, "cost_per_correct_usd": 0.000896, "output_tokens_per_s": 15.3, "time_to_first_token_s": 1.162, "end_to_end_s": 25.62, "output_tokens_per_question": 723, "context_window": 1310720, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.3, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "13/50", "mmlu-pro_correct": "20/30", "subject_science_correct": "13/30", "subject_engineering_correct": "2/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "4/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "qwen/qwen3.7-plus", "name": "Qwen3.7 Plus", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-7-plus", "open_weights": null, "reasoning": true, "score_pct": 60, "correct": 48, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.003256, "cost_per_correct_usd": 0.005427, "output_tokens_per_s": 62.5, "time_to_first_token_s": 1.141, "end_to_end_s": 12.35, "output_tokens_per_question": 2462, "context_window": 1000000, "list_input_usd_per_m": 0.32, "list_output_usd_per_m": 1.28, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "21/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "19/30", "subject_engineering_correct": "4/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "7/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "nvidia/nemotron-3.5-super-vl", "name": "Nemotron 3.5 Super VL", "creator": "NVIDIA", "url": "https://www.llmverifications.com/models/nemotron-3-5-super-vl", "open_weights": true, "reasoning": true, "score_pct": 40, "correct": 32, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001269, "cost_per_correct_usd": 0.003173, "output_tokens_per_s": 714.9, "time_to_first_token_s": 0.647, "end_to_end_s": 1.53, "output_tokens_per_question": 2771, "context_window": 1048576, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.5, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "12/50", "mmlu-pro_correct": "20/30", "subject_science_correct": "13/30", "subject_engineering_correct": "4/15", "subject_medicine_correct": "2/11", "subject_economics_correct": "6/9", "subject_law_correct": "0/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "google/gemma-4-26b-a4b-it", "name": "Gemma 4 26B A4B", "creator": "Google", "url": "https://www.llmverifications.com/models/gemma-4-26b-a4b-it", "open_weights": true, "reasoning": true, "score_pct": 15, "correct": 12, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001073, "cost_per_correct_usd": 0.00715, "output_tokens_per_s": 171.9, "time_to_first_token_s": 0.682, "end_to_end_s": 4.84, "output_tokens_per_question": 3722, "context_window": 262144, "list_input_usd_per_m": 0.09, "list_output_usd_per_m": 0.3, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "1/50", "mmlu-pro_correct": "11/30", "subject_science_correct": "4/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "1/11", "subject_economics_correct": "4/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "2/9" }, { "model_id": "google/gemma-4-31b-it", "name": "Gemma 4 31B", "creator": "Google", "url": "https://www.llmverifications.com/models/gemma-4-31b-it", "open_weights": true, "reasoning": true, "score_pct": 30, "correct": 24, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.00139, "cost_per_correct_usd": 0.004633, "output_tokens_per_s": 58.4, "time_to_first_token_s": 0.643, "end_to_end_s": 12.62, "output_tokens_per_question": 3307, "context_window": 262144, "list_input_usd_per_m": 0.2, "list_output_usd_per_m": 0.4, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "7/50", "mmlu-pro_correct": "17/30", "subject_science_correct": "11/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "2/11", "subject_economics_correct": "5/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "5/9" }, { "model_id": "inclusionai/ling-3.1-flash", "name": "Ling 3.1 Flash", "creator": "inclusionAI", "url": "https://www.llmverifications.com/models/ling-3-1-flash", "open_weights": null, "reasoning": true, "score_pct": 41.2, "correct": 33, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000613, "cost_per_correct_usd": 0.001489, "output_tokens_per_s": 7.1, "time_to_first_token_s": 1.064, "end_to_end_s": 81.35, "output_tokens_per_question": 2616, "context_window": 262144, "list_input_usd_per_m": 0.075, "list_output_usd_per_m": 0.225, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "12/50", "mmlu-pro_correct": "21/30", "subject_science_correct": "16/30", "subject_engineering_correct": "1/15", "subject_medicine_correct": "2/11", "subject_economics_correct": "6/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "qwen/qwen3.5-397b-a17b", "name": "Qwen3.5 397B A17B", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-5-397b-a17b", "open_weights": true, "reasoning": true, "score_pct": 32.5, "correct": 26, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.013352, "cost_per_correct_usd": 0.041082, "output_tokens_per_s": 239.6, "time_to_first_token_s": 0.647, "end_to_end_s": 3.53, "output_tokens_per_question": 4010, "context_window": 262144, "list_input_usd_per_m": 0.5, "list_output_usd_per_m": 3.6, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "5/50", "mmlu-pro_correct": "21/30", "subject_science_correct": "10/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "2/11", "subject_economics_correct": "5/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "nousresearch/hermes-4-405b", "name": "Hermes 4 405B", "creator": "Nous", "url": "https://www.llmverifications.com/models/hermes-4-405b", "open_weights": true, "reasoning": true, "score_pct": 26.2, "correct": 21, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.01, "cost_per_correct_usd": 0.038167, "output_tokens_per_s": 40.8, "time_to_first_token_s": 0.35, "end_to_end_s": 17.52, "output_tokens_per_question": 3223, "context_window": 131072, "list_input_usd_per_m": 1, "list_output_usd_per_m": 3, "quality_tested_at": "2026-10-27T17:31:31.658Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "3/50", "mmlu-pro_correct": "18/30", "subject_science_correct": "5/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "3/11", "subject_economics_correct": "5/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "anthropic/claude-fable-5.1", "name": "Claude Fable 5.1", "creator": "Anthropic", "url": "https://www.llmverifications.com/models/claude-fable-5-1", "open_weights": null, "reasoning": true, "score_pct": 81.2, "correct": 65, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.016054, "cost_per_correct_usd": 0.01977, "output_tokens_per_s": 70.4, "time_to_first_token_s": 2.095, "end_to_end_s": 11.04, "output_tokens_per_question": 238, "context_window": 1000000, "list_input_usd_per_m": 10, "list_output_usd_per_m": 50, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "38/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "25/30", "subject_engineering_correct": "13/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "7/9", "subject_law_correct": "5/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-6-astra", "name": "GPT-6 Astra", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-6-astra", "open_weights": null, "reasoning": true, "score_pct": 76.2, "correct": 61, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.013326, "cost_per_correct_usd": 0.017488, "output_tokens_per_s": 48.9, "time_to_first_token_s": 1.291, "end_to_end_s": 8.92, "output_tokens_per_question": 205, "context_window": 1050000, "list_input_usd_per_m": 10, "list_output_usd_per_m": 50, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "35/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "26/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "meta/muse-wonder", "name": "Muse Wonder", "creator": "Meta", "url": "https://www.llmverifications.com/models/muse-wonder", "open_weights": null, "reasoning": true, "score_pct": 78.8, "correct": 63, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.021439, "cost_per_correct_usd": 0.027206, "output_tokens_per_s": 174.9, "time_to_first_token_s": 16.063, "end_to_end_s": 20.8, "output_tokens_per_question": 1014, "context_window": 1048576, "list_input_usd_per_m": 4, "list_output_usd_per_m": 20, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "35/50", "mmlu-pro_correct": "28/30", "subject_science_correct": "26/30", "subject_engineering_correct": "11/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "x-ai/grok-4.7", "name": "Grok 4.7", "creator": "SpaceXAI", "url": "https://www.llmverifications.com/models/grok-4-7", "open_weights": null, "reasoning": true, "score_pct": 80, "correct": 64, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.025053, "cost_per_correct_usd": 0.031316, "output_tokens_per_s": 77.1, "time_to_first_token_s": 1.881, "end_to_end_s": 8, "output_tokens_per_question": 3952, "context_window": 500000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 6, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "37/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "27/30", "subject_engineering_correct": "11/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/o3", "name": "o3", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/o3", "open_weights": null, "reasoning": true, "score_pct": 72.5, "correct": 58, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.008812, "cost_per_correct_usd": 0.012155, "output_tokens_per_s": 163.1, "time_to_first_token_s": 1.232, "end_to_end_s": 3.92, "output_tokens_per_question": 1025, "context_window": 200000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 8, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "30/50", "mmlu-pro_correct": "28/30", "subject_science_correct": "23/30", "subject_engineering_correct": "8/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "7/9", "subject_law_correct": "5/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-5.6-terra", "name": "GPT-5.6 Terra", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-5-6-terra", "open_weights": null, "reasoning": true, "score_pct": 75, "correct": 60, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.006278, "cost_per_correct_usd": 0.008371, "output_tokens_per_s": 101, "time_to_first_token_s": 1.149, "end_to_end_s": 5.29, "output_tokens_per_question": 472, "context_window": 1050000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 12, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "33/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "26/30", "subject_engineering_correct": "11/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-5.3-codex", "name": "GPT-5.3-Codex", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-5-3-codex", "open_weights": null, "reasoning": true, "score_pct": 82.5, "correct": 66, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.010318, "cost_per_correct_usd": 0.012506, "output_tokens_per_s": 95.1, "time_to_first_token_s": 1.357, "end_to_end_s": 5.25, "output_tokens_per_question": 699, "context_window": 400000, "list_input_usd_per_m": 1.75, "list_output_usd_per_m": 14, "quality_tested_at": "2026-10-27T17:30:24.045Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "39/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "28/30", "subject_engineering_correct": "11/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-6.1-terra", "name": "GPT-6.1 Terra", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-6-1-terra", "open_weights": null, "reasoning": true, "score_pct": 73.8, "correct": 59, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001278, "cost_per_correct_usd": 0.001732, "output_tokens_per_s": 122.3, "time_to_first_token_s": 2.403, "end_to_end_s": 5.83, "output_tokens_per_question": 195, "context_window": 1050000, "list_input_usd_per_m": 1, "list_output_usd_per_m": 5, "quality_tested_at": "2026-10-27T10:07:30.431Z", "speed_tested_at": "2026-10-27T10:07:30.431Z", "supergpqa_correct": "33/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "26/30", "subject_engineering_correct": "9/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "7/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-6.1-terra-pro", "name": "GPT-6.1 Terra Pro", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-6-1-terra-pro", "open_weights": null, "reasoning": true, "score_pct": 76.2, "correct": 61, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.004962, "cost_per_correct_usd": 0.006512, "output_tokens_per_s": null, "time_to_first_token_s": 8.535, "end_to_end_s": 9.02, "output_tokens_per_question": 453, "context_window": 1050000, "list_input_usd_per_m": 1, "list_output_usd_per_m": 5, "quality_tested_at": "2026-10-27T10:07:30.431Z", "speed_tested_at": "2026-10-27T10:07:30.431Z", "supergpqa_correct": "35/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "26/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-oss-20b", "name": "gpt-oss-20b", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-oss-20b", "open_weights": true, "reasoning": true, "score_pct": 51.2, "correct": 41, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000066, "cost_per_correct_usd": 0.000128, "output_tokens_per_s": 102.8, "time_to_first_token_s": 0.69, "end_to_end_s": 4.03, "output_tokens_per_question": 472, "context_window": 131072, "list_input_usd_per_m": 0.018, "list_output_usd_per_m": 0.09, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "18/50", "mmlu-pro_correct": "23/30", "subject_science_correct": "18/30", "subject_engineering_correct": "4/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "6/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "ibm-granite/granite-4.0-h-micro", "name": "Granite 4.0 Micro", "creator": "IBM", "url": "https://www.llmverifications.com/models/granite-4-0-h-micro", "open_weights": true, "reasoning": false, "score_pct": 25, "correct": 20, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000051, "cost_per_correct_usd": 0.000203, "output_tokens_per_s": 38.2, "time_to_first_token_s": 0.697, "end_to_end_s": 8.89, "output_tokens_per_question": 403, "context_window": 131000, "list_input_usd_per_m": 0.017, "list_output_usd_per_m": 0.112, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "8/50", "mmlu-pro_correct": "12/30", "subject_science_correct": "7/30", "subject_engineering_correct": "1/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "2/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "5/9" }, { "model_id": "mistralai/ministral-3b-2512", "name": "Ministral 3 3B 2512", "creator": "Mistral", "url": "https://www.llmverifications.com/models/ministral-3b-2512", "open_weights": true, "reasoning": false, "score_pct": 26.2, "correct": 21, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000195, "cost_per_correct_usd": 0.000743, "output_tokens_per_s": 251.9, "time_to_first_token_s": 0.305, "end_to_end_s": 1.61, "output_tokens_per_question": 1652, "context_window": 131072, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.1, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "7/50", "mmlu-pro_correct": "14/30", "subject_science_correct": "5/30", "subject_engineering_correct": "2/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "4/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "5/9" }, { "model_id": "amazon/nova-micro-v1", "name": "Nova Micro 1.0", "creator": "Amazon", "url": "https://www.llmverifications.com/models/nova-micro-v1", "open_weights": null, "reasoning": false, "score_pct": 22.5, "correct": 18, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000087, "cost_per_correct_usd": 0.000387, "output_tokens_per_s": 179.9, "time_to_first_token_s": 0.467, "end_to_end_s": 2.11, "output_tokens_per_question": 545, "context_window": 128000, "list_input_usd_per_m": 0.035, "list_output_usd_per_m": 0.14, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "3/50", "mmlu-pro_correct": "15/30", "subject_science_correct": "3/30", "subject_engineering_correct": "1/15", "subject_medicine_correct": "3/11", "subject_economics_correct": "6/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "4/9" }, { "model_id": "inception/mercury-2.5", "name": "Mercury 2.5", "creator": "Inception", "url": "https://www.llmverifications.com/models/mercury-2-5", "open_weights": null, "reasoning": true, "score_pct": 58.2, "correct": 46, "graded": 79, "failed_requests": 1, "cost_per_question_usd": 0.000113, "cost_per_correct_usd": 0.000195, "output_tokens_per_s": 157.3, "time_to_first_token_s": 2.252, "end_to_end_s": 6.5, "output_tokens_per_question": 673, "context_window": 260000, "list_input_usd_per_m": 0.04, "list_output_usd_per_m": 0.15, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "21/50", "mmlu-pro_correct": "25/29", "subject_science_correct": "19/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "6/8" }, { "model_id": "microsoft/phi-4", "name": "Phi 4", "creator": "Microsoft", "url": "https://www.llmverifications.com/models/phi-4", "open_weights": true, "reasoning": false, "score_pct": 41.2, "correct": 33, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000119, "cost_per_correct_usd": 0.00029, "output_tokens_per_s": 102.9, "time_to_first_token_s": 0.466, "end_to_end_s": 4.88, "output_tokens_per_question": 699, "context_window": 16384, "list_input_usd_per_m": 0.07, "list_output_usd_per_m": 0.14, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "12/50", "mmlu-pro_correct": "21/30", "subject_science_correct": "12/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "5/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "4/9" }, { "model_id": "upstage/solar-mini4", "name": "Solar Mini 4", "creator": "Upstage", "url": "https://www.llmverifications.com/models/solar-mini4", "open_weights": null, "reasoning": true, "score_pct": 41.2, "correct": 33, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000527, "cost_per_correct_usd": 0.00128, "output_tokens_per_s": 107, "time_to_first_token_s": 0.693, "end_to_end_s": 7.43, "output_tokens_per_question": 2552, "context_window": 524288, "list_input_usd_per_m": 0.05, "list_output_usd_per_m": 0.2, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "13/50", "mmlu-pro_correct": "20/30", "subject_science_correct": "16/30", "subject_engineering_correct": "2/15", "subject_medicine_correct": "2/11", "subject_economics_correct": "7/9", "subject_law_correct": "0/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "mistralai/ministral-8b-2512", "name": "Ministral 3 8B 2512", "creator": "Mistral", "url": "https://www.llmverifications.com/models/ministral-8b-2512", "open_weights": true, "reasoning": false, "score_pct": 42.5, "correct": 34, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000269, "cost_per_correct_usd": 0.000634, "output_tokens_per_s": 123.8, "time_to_first_token_s": 0.324, "end_to_end_s": 3.5, "output_tokens_per_question": 1496, "context_window": 262144, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.15, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "10/50", "mmlu-pro_correct": "24/30", "subject_science_correct": "10/30", "subject_engineering_correct": "4/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "7/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "nvidia/nemotron-3.5-lightning", "name": "Nemotron 3.5 Lightning", "creator": "NVIDIA", "url": "https://www.llmverifications.com/models/nemotron-3-5-lightning", "open_weights": true, "reasoning": true, "score_pct": 43.8, "correct": 35, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000489, "cost_per_correct_usd": 0.001117, "output_tokens_per_s": 116.3, "time_to_first_token_s": 0.495, "end_to_end_s": 6.52, "output_tokens_per_question": 2846, "context_window": 262144, "list_input_usd_per_m": 0.07, "list_output_usd_per_m": 0.2, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "16/50", "mmlu-pro_correct": "19/30", "subject_science_correct": "14/30", "subject_engineering_correct": "4/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "6/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "ibm-granite/granite-4.2-8b", "name": "Granite 4.2 8B", "creator": "IBM", "url": "https://www.llmverifications.com/models/granite-4-2-8b", "open_weights": true, "reasoning": true, "score_pct": 36.2, "correct": 29, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000077, "cost_per_correct_usd": 0.000213, "output_tokens_per_s": 59.6, "time_to_first_token_s": 0.632, "end_to_end_s": 7.34, "output_tokens_per_question": 227, "context_window": 131072, "list_input_usd_per_m": 0.06, "list_output_usd_per_m": 0.25, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "11/50", "mmlu-pro_correct": "18/30", "subject_science_correct": "13/30", "subject_engineering_correct": "4/15", "subject_medicine_correct": "1/11", "subject_economics_correct": "5/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "4/9" }, { "model_id": "mistralai/ministral-14b-2512", "name": "Ministral 3 14B 2512", "creator": "Mistral", "url": "https://www.llmverifications.com/models/ministral-14b-2512", "open_weights": true, "reasoning": false, "score_pct": 33.8, "correct": 27, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000311, "cost_per_correct_usd": 0.000921, "output_tokens_per_s": 32.2, "time_to_first_token_s": 0.648, "end_to_end_s": 12.74, "output_tokens_per_question": 1255, "context_window": 262144, "list_input_usd_per_m": 0.2, "list_output_usd_per_m": 0.2, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "8/50", "mmlu-pro_correct": "19/30", "subject_science_correct": "8/30", "subject_engineering_correct": "2/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "4/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "5/9" }, { "model_id": "xiaomi/mimo-v2.6-flash", "name": "MiMo-V2.6-Flash", "creator": "Xiaomi", "url": "https://www.llmverifications.com/models/mimo-v2-6-flash", "open_weights": true, "reasoning": true, "score_pct": 62.5, "correct": 50, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.00056, "cost_per_correct_usd": 0.000897, "output_tokens_per_s": 77.2, "time_to_first_token_s": 0.414, "end_to_end_s": 5.38, "output_tokens_per_question": 1850, "context_window": 1050000, "list_input_usd_per_m": 0.14, "list_output_usd_per_m": 0.28, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "27/50", "mmlu-pro_correct": "23/30", "subject_science_correct": "20/30", "subject_engineering_correct": "7/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "7/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "meta-llama/llama-3.3-70b-instruct", "name": "Llama 3.3 70B Instruct", "creator": "Meta", "url": "https://www.llmverifications.com/models/llama-3-3-70b-instruct", "open_weights": true, "reasoning": false, "score_pct": 42.5, "correct": 34, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000535, "cost_per_correct_usd": 0.001258, "output_tokens_per_s": 41.8, "time_to_first_token_s": 0.678, "end_to_end_s": 8.82, "output_tokens_per_question": 855, "context_window": 131072, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.32, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "9/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "11/30", "subject_engineering_correct": "2/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "5/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "tencent/hy3", "name": "Hy3", "creator": "Tencent", "url": "https://www.llmverifications.com/models/hy3", "open_weights": true, "reasoning": true, "score_pct": 58.8, "correct": 47, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001237, "cost_per_correct_usd": 0.002103, "output_tokens_per_s": 95.7, "time_to_first_token_s": 1.702, "end_to_end_s": 9.02, "output_tokens_per_question": 2355, "context_window": 262144, "list_input_usd_per_m": 0.132, "list_output_usd_per_m": 0.528, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "22/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "20/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "z-ai/glm-5.3-flash", "name": "GLM 5.3 Flash", "creator": "Z.ai", "url": "https://www.llmverifications.com/models/glm-5-3-flash", "open_weights": true, "reasoning": true, "score_pct": 67.5, "correct": 54, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000146, "cost_per_correct_usd": 0.000216, "output_tokens_per_s": 83.2, "time_to_first_token_s": 0.497, "end_to_end_s": 7.39, "output_tokens_per_question": 560, "context_window": 1048576, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.5, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "27/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "22/30", "subject_engineering_correct": "8/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "upstage/solar-pro4", "name": "Solar Pro 4", "creator": "Upstage", "url": "https://www.llmverifications.com/models/solar-pro4", "open_weights": null, "reasoning": true, "score_pct": 45, "correct": 36, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001823, "cost_per_correct_usd": 0.00405, "output_tokens_per_s": 91.1, "time_to_first_token_s": 0.606, "end_to_end_s": 8.32, "output_tokens_per_question": 2952, "context_window": 524288, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.6, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "14/50", "mmlu-pro_correct": "22/30", "subject_science_correct": "18/30", "subject_engineering_correct": "1/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "5/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "mistralai/mistral-small-2603", "name": "Mistral Small 4", "creator": "Mistral", "url": "https://www.llmverifications.com/models/mistral-small-2603", "open_weights": true, "reasoning": true, "score_pct": 32.5, "correct": 26, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001859, "cost_per_correct_usd": 0.00572, "output_tokens_per_s": 170.2, "time_to_first_token_s": 0.482, "end_to_end_s": 4.51, "output_tokens_per_question": 3021, "context_window": 262144, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.6, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "7/50", "mmlu-pro_correct": "19/30", "subject_science_correct": "7/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "6/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "deepseek/deepseek-v4-flash-vision-exp", "name": "DeepSeek V4 Flash Vision Exp", "creator": "DeepSeek", "url": "https://www.llmverifications.com/models/deepseek-v4-flash-vision-exp", "open_weights": true, "reasoning": true, "score_pct": 71.2, "correct": 57, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001066, "cost_per_correct_usd": 0.001498, "output_tokens_per_s": 170.1, "time_to_first_token_s": 0.737, "end_to_end_s": 3.18, "output_tokens_per_question": 1301, "context_window": 1048576, "list_input_usd_per_m": 0.2156, "list_output_usd_per_m": 0.6468, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "31/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "25/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "8/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "qwen/qwen3-coder-next", "name": "Qwen3 Coder Next", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-coder-next", "open_weights": true, "reasoning": false, "score_pct": 62.5, "correct": 50, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001686, "cost_per_correct_usd": 0.002697, "output_tokens_per_s": 106.1, "time_to_first_token_s": 0.425, "end_to_end_s": 4.19, "output_tokens_per_question": 2059, "context_window": 262144, "list_input_usd_per_m": 0.12, "list_output_usd_per_m": 0.8, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "23/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "19/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "arcee-ai/trinity-large-thinking", "name": "Trinity Large Thinking", "creator": "Arcee AI", "url": "https://www.llmverifications.com/models/trinity-large-thinking", "open_weights": null, "reasoning": true, "score_pct": 28.8, "correct": 23, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.002507, "cost_per_correct_usd": 0.008704, "output_tokens_per_s": 328.7, "time_to_first_token_s": 0.566, "end_to_end_s": 2.73, "output_tokens_per_question": 3043, "context_window": 262144, "list_input_usd_per_m": 0.25, "list_output_usd_per_m": 0.8, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "5/50", "mmlu-pro_correct": "18/30", "subject_science_correct": "6/30", "subject_engineering_correct": "0/15", "subject_medicine_correct": "3/11", "subject_economics_correct": "6/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "qwen/qwen3.6-35b-a3b", "name": "Qwen3.6 35B A3B", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-6-35b-a3b", "open_weights": true, "reasoning": true, "score_pct": 37.5, "correct": 30, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.003724, "cost_per_correct_usd": 0.00993, "output_tokens_per_s": 134.8, "time_to_first_token_s": 0.424, "end_to_end_s": 5.62, "output_tokens_per_question": 3563, "context_window": 262144, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.95, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "9/50", "mmlu-pro_correct": "21/30", "subject_science_correct": "9/30", "subject_engineering_correct": "2/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "5/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "minimax/minimax-m3", "name": "MiniMax M3", "creator": "MiniMax", "url": "https://www.llmverifications.com/models/minimax-m3", "open_weights": true, "reasoning": true, "score_pct": 57.5, "correct": 46, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.002568, "cost_per_correct_usd": 0.004467, "output_tokens_per_s": 121, "time_to_first_token_s": 0.517, "end_to_end_s": 4.23, "output_tokens_per_question": 2129, "context_window": 1048576, "list_input_usd_per_m": 0.3, "list_output_usd_per_m": 1.2, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "23/50", "mmlu-pro_correct": "23/30", "subject_science_correct": "24/30", "subject_engineering_correct": "5/15", "subject_medicine_correct": "3/11", "subject_economics_correct": "6/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "5/9" }, { "model_id": "meituan/longcat-2.0", "name": "LongCat 2.0", "creator": "Meituan", "url": "https://www.llmverifications.com/models/longcat-2-0", "open_weights": true, "reasoning": true, "score_pct": 48.8, "correct": 39, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.003209, "cost_per_correct_usd": 0.006576, "output_tokens_per_s": 66.8, "time_to_first_token_s": 1.363, "end_to_end_s": 9.99, "output_tokens_per_question": 2600, "context_window": 1048756, "list_input_usd_per_m": 0.3, "list_output_usd_per_m": 1.2, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "17/50", "mmlu-pro_correct": "22/30", "subject_science_correct": "18/30", "subject_engineering_correct": "1/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "thinkingmachines/inkling-small", "name": "Inkling Small", "creator": "Thinking Machines", "url": "https://www.llmverifications.com/models/inkling-small", "open_weights": true, "reasoning": true, "score_pct": 67.5, "correct": 54, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000468, "cost_per_correct_usd": 0.000694, "output_tokens_per_s": 233.4, "time_to_first_token_s": 0.512, "end_to_end_s": 3.59, "output_tokens_per_question": 273, "context_window": 524288, "list_input_usd_per_m": 0.45, "list_output_usd_per_m": 1.2, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "28/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "24/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "qwen/qwen3.8-27b", "name": "Qwen3.8 27B", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-8-27b", "open_weights": true, "reasoning": true, "score_pct": 63.8, "correct": 51, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.004303, "cost_per_correct_usd": 0.006745, "output_tokens_per_s": 48.5, "time_to_first_token_s": 0.528, "end_to_end_s": 10.59, "output_tokens_per_question": 1913, "context_window": 1000000, "list_input_usd_per_m": 0.2, "list_output_usd_per_m": 2.5, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "28/50", "mmlu-pro_correct": "23/30", "subject_science_correct": "21/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "google/gemini-3.5-flash-lite", "name": "Gemini 3.5 Flash Lite", "creator": "Google", "url": "https://www.llmverifications.com/models/gemini-3-5-flash-lite", "open_weights": null, "reasoning": true, "score_pct": 75, "correct": 60, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.003371, "cost_per_correct_usd": 0.004495, "output_tokens_per_s": 409.1, "time_to_first_token_s": 1.067, "end_to_end_s": 2.77, "output_tokens_per_question": 1311, "context_window": 1048576, "list_input_usd_per_m": 0.3, "list_output_usd_per_m": 2.5, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "33/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "23/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "nvidia/nemotron-3-ultra-550b-a55b", "name": "Nemotron 3 Ultra", "creator": "NVIDIA", "url": "https://www.llmverifications.com/models/nemotron-3-ultra-550b-a55b", "open_weights": true, "reasoning": true, "score_pct": 60, "correct": 48, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.00566, "cost_per_correct_usd": 0.009433, "output_tokens_per_s": 135.8, "time_to_first_token_s": 0.805, "end_to_end_s": 3.65, "output_tokens_per_question": 2177, "context_window": 262144, "list_input_usd_per_m": 0.6, "list_output_usd_per_m": 2.4, "quality_tested_at": "2026-10-26T05:22:44.688Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "23/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "22/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "6/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "liquid/lfm-2.5-2.6b", "name": "LFM2.5-2.6B", "creator": "LiquidAI", "url": "https://www.llmverifications.com/models/lfm-2-5-2-6b", "open_weights": true, "reasoning": true, "score_pct": null, "correct": null, "graded": null, "failed_requests": 80, "cost_per_question_usd": null, "cost_per_correct_usd": null, "output_tokens_per_s": null, "time_to_first_token_s": null, "end_to_end_s": null, "output_tokens_per_question": null, "context_window": 128000, "list_input_usd_per_m": 0.35, "list_output_usd_per_m": 0.5, "quality_tested_at": "2026-10-26T04:44:39.525Z", "speed_tested_at": "2026-10-26T04:44:39.525Z", "supergpqa_correct": null, "mmlu-pro_correct": null, "subject_science_correct": null, "subject_engineering_correct": null, "subject_medicine_correct": null, "subject_economics_correct": null, "subject_law_correct": null, "subject_humanities_social_science_correct": null }, { "model_id": "google/gemini-3.1-pro-preview", "name": "Gemini 3.1 Pro Preview", "creator": "Google", "url": "https://www.llmverifications.com/models/gemini-3-1-pro-preview", "open_weights": null, "reasoning": true, "score_pct": 77.5, "correct": 62, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.017202, "cost_per_correct_usd": 0.022195, "output_tokens_per_s": 124, "time_to_first_token_s": 3.43, "end_to_end_s": 8.9, "output_tokens_per_question": 1382, "context_window": 1048576, "list_input_usd_per_m": 2, "list_output_usd_per_m": 12, "quality_tested_at": "2026-10-26T03:44:50.952Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "36/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "26/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "deepseek/deepseek-v4-pro-0813", "name": "DeepSeek V4 Pro 0813", "creator": "DeepSeek", "url": "https://www.llmverifications.com/models/deepseek-v4-pro-0813", "open_weights": true, "reasoning": true, "score_pct": 71.2, "correct": 57, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.003617, "cost_per_correct_usd": 0.00508, "output_tokens_per_s": 56.1, "time_to_first_token_s": 0.81, "end_to_end_s": 10.07, "output_tokens_per_question": 1318, "context_window": 1048576, "list_input_usd_per_m": 1.32, "list_output_usd_per_m": 3.96, "quality_tested_at": "2026-10-26T03:44:50.952Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "30/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "25/30", "subject_engineering_correct": "8/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "mistralai/mistral-large-4-0", "name": "Mistral Large 4", "creator": "Mistral", "url": "https://www.llmverifications.com/models/mistral-large-4-0", "open_weights": null, "reasoning": true, "score_pct": 50, "correct": 40, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.011316, "cost_per_correct_usd": 0.022631, "output_tokens_per_s": 83.2, "time_to_first_token_s": 1.055, "end_to_end_s": 9.49, "output_tokens_per_question": 2613, "context_window": 1048576, "list_input_usd_per_m": 1.36, "list_output_usd_per_m": 4.18, "quality_tested_at": "2026-10-26T03:44:50.952Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "17/50", "mmlu-pro_correct": "23/30", "subject_science_correct": "20/30", "subject_engineering_correct": "3/15", "subject_medicine_correct": "2/11", "subject_economics_correct": "7/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "meta/muse-spark-open", "name": "Muse Spark Open", "creator": "Meta", "url": "https://www.llmverifications.com/models/muse-spark-open", "open_weights": true, "reasoning": true, "score_pct": 77.5, "correct": 62, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.00452, "cost_per_correct_usd": 0.005832, "output_tokens_per_s": 220.1, "time_to_first_token_s": 0.496, "end_to_end_s": 3.25, "output_tokens_per_question": 964, "context_window": 1048576, "list_input_usd_per_m": 1.25, "list_output_usd_per_m": 4.25, "quality_tested_at": "2026-10-26T03:44:50.952Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "35/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "25/30", "subject_engineering_correct": "11/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "anthropic/claude-opus-5.5", "name": "Claude Opus 5.5", "creator": "Anthropic", "url": "https://www.llmverifications.com/models/claude-opus-5-5", "open_weights": null, "reasoning": true, "score_pct": 85, "correct": 68, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.01465, "cost_per_correct_usd": 0.017235, "output_tokens_per_s": 129.4, "time_to_first_token_s": 2.34, "end_to_end_s": 7.58, "output_tokens_per_question": 645, "context_window": 1000000, "list_input_usd_per_m": 4, "list_output_usd_per_m": 20, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "40/50", "mmlu-pro_correct": "28/30", "subject_science_correct": "28/30", "subject_engineering_correct": "12/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "8/9", "subject_law_correct": "5/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "google/gemini-3.8-flash", "name": "Gemini 3.8 Flash", "creator": "Google", "url": "https://www.llmverifications.com/models/gemini-3-8-flash", "open_weights": null, "reasoning": true, "score_pct": 82.5, "correct": 66, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.003703, "cost_per_correct_usd": 0.004488, "output_tokens_per_s": 142.2, "time_to_first_token_s": 3.173, "end_to_end_s": 5.64, "output_tokens_per_question": 925, "context_window": 1048576, "list_input_usd_per_m": 0.825, "list_output_usd_per_m": 4.125, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "37/50", "mmlu-pro_correct": "29/30", "subject_science_correct": "26/30", "subject_engineering_correct": "12/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "6/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "z-ai/glm-5.3", "name": "GLM 5.3", "creator": "Z.ai", "url": "https://www.llmverifications.com/models/glm-5-3", "open_weights": true, "reasoning": true, "score_pct": 67.5, "correct": 54, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000933, "cost_per_correct_usd": 0.001382, "output_tokens_per_s": 153.7, "time_to_first_token_s": 0.763, "end_to_end_s": 3.35, "output_tokens_per_question": 523, "context_window": 1048576, "list_input_usd_per_m": 0.079, "list_output_usd_per_m": 1.4, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "27/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "20/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "deepseek/deepseek-v4.1-flash-1017", "name": "DeepSeek V4.1 Flash 1017", "creator": "DeepSeek", "url": "https://www.llmverifications.com/models/deepseek-v4-1-flash-1017", "open_weights": true, "reasoning": true, "score_pct": 73.8, "correct": 59, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000906, "cost_per_correct_usd": 0.001227, "output_tokens_per_s": 57.4, "time_to_first_token_s": 0.877, "end_to_end_s": 7.31, "output_tokens_per_question": 1318, "context_window": 1048576, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.6, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "31/50", "mmlu-pro_correct": "28/30", "subject_science_correct": "26/30", "subject_engineering_correct": "9/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "openai/gpt-6.1-sol", "name": "GPT-6.1 Sol", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-6-1-sol", "open_weights": null, "reasoning": true, "score_pct": 76.2, "correct": 61, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.002855, "cost_per_correct_usd": 0.003746, "output_tokens_per_s": 41.2, "time_to_first_token_s": 2.099, "end_to_end_s": 10.81, "output_tokens_per_question": 224, "context_window": 1050000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 10, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "35/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "27/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "google/gemini-4-argon", "name": "Gemini 4 Argon", "creator": "Google", "url": "https://www.llmverifications.com/models/gemini-4-argon", "open_weights": null, "reasoning": true, "score_pct": 84.8, "correct": 67, "graded": 79, "failed_requests": 1, "cost_per_question_usd": 0.006261, "cost_per_correct_usd": 0.007383, "output_tokens_per_s": 68.9, "time_to_first_token_s": 3.29, "end_to_end_s": 9.88, "output_tokens_per_question": 558, "context_window": 1048576, "list_input_usd_per_m": 2, "list_output_usd_per_m": 10, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "38/49", "mmlu-pro_correct": "29/30", "subject_science_correct": "25/30", "subject_engineering_correct": "13/14", "subject_medicine_correct": "7/11", "subject_economics_correct": "8/9", "subject_law_correct": "6/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-5.4", "name": "GPT-5.4", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-5-4", "open_weights": null, "reasoning": true, "score_pct": 72.5, "correct": 58, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.011346, "cost_per_correct_usd": 0.015649, "output_tokens_per_s": 55.2, "time_to_first_token_s": 1.717, "end_to_end_s": 8.73, "output_tokens_per_question": 706, "context_window": 1050000, "list_input_usd_per_m": 2.5, "list_output_usd_per_m": 15, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "32/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "25/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "anthropic/claude-haiku-5.5", "name": "Claude Haiku 5.5", "creator": "Anthropic", "url": "https://www.llmverifications.com/models/claude-haiku-5-5", "open_weights": null, "reasoning": true, "score_pct": 68.8, "correct": 55, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.00048, "cost_per_correct_usd": 0.000698, "output_tokens_per_s": 191.7, "time_to_first_token_s": 0.677, "end_to_end_s": 3.92, "output_tokens_per_question": 873, "context_window": 1000000, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.5, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "30/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "23/30", "subject_engineering_correct": "10/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "8/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "openai/gpt-6-sol", "name": "GPT-6 Sol", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-6-sol", "open_weights": null, "reasoning": true, "score_pct": 73.8, "correct": 59, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.002355, "cost_per_correct_usd": 0.003191, "output_tokens_per_s": 92.3, "time_to_first_token_s": 0.909, "end_to_end_s": 4.35, "output_tokens_per_question": 174, "context_window": 1050000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 10, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "33/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "27/30", "subject_engineering_correct": "8/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "7/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-6-luna", "name": "GPT-6 Luna", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-6-luna", "open_weights": null, "reasoning": true, "score_pct": 72.5, "correct": 58, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000246, "cost_per_correct_usd": 0.00034, "output_tokens_per_s": 136.9, "time_to_first_token_s": 1.196, "end_to_end_s": 4.26, "output_tokens_per_question": 431, "context_window": 1050000, "list_input_usd_per_m": 0.1, "list_output_usd_per_m": 0.5, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "33/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "25/30", "subject_engineering_correct": "9/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "7/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "minimax/minimax-m3.1-flash", "name": "MiniMax M3.1 Flash", "creator": "MiniMax", "url": "https://www.llmverifications.com/models/minimax-m3-1-flash", "open_weights": true, "reasoning": true, "score_pct": 66.2, "correct": 53, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000868, "cost_per_correct_usd": 0.001312, "output_tokens_per_s": 130.5, "time_to_first_token_s": 0.526, "end_to_end_s": 4.17, "output_tokens_per_question": 1376, "context_window": 1048576, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.6, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "28/50", "mmlu-pro_correct": "25/30", "subject_science_correct": "25/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "5/11", "subject_economics_correct": "8/9", "subject_law_correct": "2/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "xiaomi/mimo-v2.6-pro", "name": "MiMo-V2.6-Pro", "creator": "Xiaomi", "url": "https://www.llmverifications.com/models/mimo-v2-6-pro", "open_weights": true, "reasoning": true, "score_pct": 68.8, "correct": 55, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.001967, "cost_per_correct_usd": 0.002859, "output_tokens_per_s": 58.1, "time_to_first_token_s": 0.668, "end_to_end_s": 7.92, "output_tokens_per_question": 2157, "context_window": 1050000, "list_input_usd_per_m": 0.435, "list_output_usd_per_m": 0.87, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "28/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "22/30", "subject_engineering_correct": "8/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "openai/gpt-oss-120b", "name": "gpt-oss-120b", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-oss-120b", "open_weights": true, "reasoning": true, "score_pct": 58.8, "correct": 47, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.000126, "cost_per_correct_usd": 0.000214, "output_tokens_per_s": 81.8, "time_to_first_token_s": 0.529, "end_to_end_s": 5.43, "output_tokens_per_question": 546, "context_window": 131072, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 0.6, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "25/50", "mmlu-pro_correct": "22/30", "subject_science_correct": "23/30", "subject_engineering_correct": "6/15", "subject_medicine_correct": "4/11", "subject_economics_correct": "7/9", "subject_law_correct": "1/6", "subject_humanities_social_science_correct": "6/9" }, { "model_id": "meta-llama/llama-4-maverick", "name": "Llama 4 Maverick", "creator": "Meta", "url": "https://www.llmverifications.com/models/llama-4-maverick", "open_weights": true, "reasoning": false, "score_pct": 59, "correct": 46, "graded": 78, "failed_requests": 2, "cost_per_question_usd": 0.000587, "cost_per_correct_usd": 0.000995, "output_tokens_per_s": 13.4, "time_to_first_token_s": 0.665, "end_to_end_s": 27.93, "output_tokens_per_question": 784, "context_window": 524288, "list_input_usd_per_m": 0.1875, "list_output_usd_per_m": 0.6525, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": "23/49", "mmlu-pro_correct": "23/29", "subject_science_correct": "18/30", "subject_engineering_correct": "6/14", "subject_medicine_correct": "5/10", "subject_economics_correct": "7/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "7/9" }, { "model_id": "moonshotai/kimi-k3.1-pro", "name": "Kimi K3.1 Pro", "creator": "MoonshotAI", "url": "https://www.llmverifications.com/models/kimi-k3-1-pro", "open_weights": null, "reasoning": true, "score_pct": 77.5, "correct": 62, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.010484, "cost_per_correct_usd": 0.013528, "output_tokens_per_s": 38.1, "time_to_first_token_s": 1.313, "end_to_end_s": 19.62, "output_tokens_per_question": 973, "context_window": 1048576, "list_input_usd_per_m": 2, "list_output_usd_per_m": 10, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "36/50", "mmlu-pro_correct": "26/30", "subject_science_correct": "26/30", "subject_engineering_correct": "11/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "qwen/qwen3.8-max-0902", "name": "Qwen3.8 Max (0902)", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-8-max-0902", "open_weights": null, "reasoning": true, "score_pct": 75, "correct": 60, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.011063, "cost_per_correct_usd": 0.014751, "output_tokens_per_s": 46, "time_to_first_token_s": 1.163, "end_to_end_s": 12.22, "output_tokens_per_question": 1722, "context_window": 1000000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 6, "quality_tested_at": "2026-10-25T17:56:53.343Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "32/50", "mmlu-pro_correct": "28/30", "subject_science_correct": "24/30", "subject_engineering_correct": "9/15", "subject_medicine_correct": "6/11", "subject_economics_correct": "8/9", "subject_law_correct": "4/6", "subject_humanities_social_science_correct": "9/9" }, { "model_id": "anthropic/claude-sonnet-5.5", "name": "Claude Sonnet 5.5", "creator": "Anthropic", "url": "https://www.llmverifications.com/models/claude-sonnet-5-5", "open_weights": null, "reasoning": true, "score_pct": 78.8, "correct": 63, "graded": 80, "failed_requests": 0, "cost_per_question_usd": 0.005434, "cost_per_correct_usd": 0.006895, "output_tokens_per_s": 126.2, "time_to_first_token_s": 0.732, "end_to_end_s": 6.22, "output_tokens_per_question": 456, "context_window": 1000000, "list_input_usd_per_m": 2, "list_output_usd_per_m": 10, "quality_tested_at": "2026-10-25T17:06:52.146Z", "speed_tested_at": "2026-10-25T15:04:38.754Z", "supergpqa_correct": "36/50", "mmlu-pro_correct": "27/30", "subject_science_correct": "24/30", "subject_engineering_correct": "13/15", "subject_medicine_correct": "7/11", "subject_economics_correct": "8/9", "subject_law_correct": "3/6", "subject_humanities_social_science_correct": "8/9" }, { "model_id": "inclusionai/ling-3.0-flash-vl", "name": "Ling 3.0 Flash VL", "creator": "inclusionAI", "url": "https://www.llmverifications.com/models/ling-3-0-flash-vl", "open_weights": true, "reasoning": true, "score_pct": null, "correct": null, "graded": null, "failed_requests": 0, "cost_per_question_usd": null, "cost_per_correct_usd": null, "output_tokens_per_s": null, "time_to_first_token_s": null, "end_to_end_s": null, "output_tokens_per_question": null, "context_window": 131072, "list_input_usd_per_m": 0.06, "list_output_usd_per_m": 0.18, "quality_tested_at": "2026-10-25T15:26:31.948Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": null, "mmlu-pro_correct": null, "subject_science_correct": null, "subject_engineering_correct": null, "subject_medicine_correct": null, "subject_economics_correct": null, "subject_law_correct": null, "subject_humanities_social_science_correct": null }, { "model_id": "qwen/qwen3.5-122b-a10b", "name": "Qwen3.5-122B-A10B", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-5-122b-a10b", "open_weights": true, "reasoning": true, "score_pct": null, "correct": null, "graded": null, "failed_requests": 0, "cost_per_question_usd": null, "cost_per_correct_usd": null, "output_tokens_per_s": 107.6, "time_to_first_token_s": 1.244, "end_to_end_s": 18.01, "output_tokens_per_question": null, "context_window": 262144, "list_input_usd_per_m": 0.4, "list_output_usd_per_m": 3.2, "quality_tested_at": "2026-10-25T15:26:31.948Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": null, "mmlu-pro_correct": null, "subject_science_correct": null, "subject_engineering_correct": null, "subject_medicine_correct": null, "subject_economics_correct": null, "subject_law_correct": null, "subject_humanities_social_science_correct": null }, { "model_id": "qwen/qwen3-next-80b-a3b-thinking", "name": "Qwen3 Next 80B A3B Thinking", "creator": "Qwen", "url": "https://www.llmverifications.com/models/qwen3-next-80b-a3b-thinking", "open_weights": true, "reasoning": true, "score_pct": null, "correct": null, "graded": null, "failed_requests": 0, "cost_per_question_usd": null, "cost_per_correct_usd": null, "output_tokens_per_s": null, "time_to_first_token_s": null, "end_to_end_s": null, "output_tokens_per_question": null, "context_window": 262144, "list_input_usd_per_m": 0.15, "list_output_usd_per_m": 1.2, "quality_tested_at": "2026-10-25T15:26:31.948Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": null, "mmlu-pro_correct": null, "subject_science_correct": null, "subject_engineering_correct": null, "subject_medicine_correct": null, "subject_economics_correct": null, "subject_law_correct": null, "subject_humanities_social_science_correct": null }, { "model_id": "amazon/nova-premier-v1", "name": "Nova Premier 1.0", "creator": "Amazon", "url": "https://www.llmverifications.com/models/nova-premier-v1", "open_weights": null, "reasoning": false, "score_pct": null, "correct": null, "graded": null, "failed_requests": 0, "cost_per_question_usd": null, "cost_per_correct_usd": null, "output_tokens_per_s": null, "time_to_first_token_s": null, "end_to_end_s": null, "output_tokens_per_question": null, "context_window": 1000000, "list_input_usd_per_m": 2.5, "list_output_usd_per_m": 12.5, "quality_tested_at": "2026-10-25T15:26:31.948Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": null, "mmlu-pro_correct": null, "subject_science_correct": null, "subject_engineering_correct": null, "subject_medicine_correct": null, "subject_economics_correct": null, "subject_law_correct": null, "subject_humanities_social_science_correct": null }, { "model_id": "openai/gpt-5.5-pro", "name": "GPT-5.5 Pro", "creator": "OpenAI", "url": "https://www.llmverifications.com/models/gpt-5-5-pro", "open_weights": null, "reasoning": true, "score_pct": null, "correct": null, "graded": null, "failed_requests": 0, "cost_per_question_usd": null, "cost_per_correct_usd": null, "output_tokens_per_s": null, "time_to_first_token_s": 33.85, "end_to_end_s": 35.39, "output_tokens_per_question": null, "context_window": 1050000, "list_input_usd_per_m": 30, "list_output_usd_per_m": 180, "quality_tested_at": "2026-10-25T15:26:31.948Z", "speed_tested_at": "2026-10-25T15:26:31.948Z", "supergpqa_correct": null, "mmlu-pro_correct": null, "subject_science_correct": null, "subject_engineering_correct": null, "subject_medicine_correct": null, "subject_economics_correct": null, "subject_law_correct": null, "subject_humanities_social_science_correct": null } ] }