{ "collectedAt": "2026-10-01", "revises": "data/research/scores-aa-round4-2026-09-22.json", "revisionNote": "Full fresh snapshot 2026-10-01 of the AA models leaderboard and coding-agents page, replacing round4 for aa_intelligence_index and aa_coding_agent_index, plus the first full aa_terminal_bench_4 snapshot from the same models payload (terminalBench40). Supersedes the AA rows of the no-baseSnapshot supplements (stepfun-step5, grok47, mimo-v26-grok47, new-models, gpt6sol, gpt6luna); those values reappear natively here.", "extraction": { "method": "curl download of public HTML on 2026-10-01; RSC embedded payload extracted with the round4 extract_all.py parser (dedup by slug / config id).", "tempDir": "_build/aa-update-round5/", "pages": { "modelsLeaderboard": "https://artificialanalysis.ai/leaderboards/models", "codingAgents": "https://artificialanalysis.ai/agents/coding-agents", "home": "https://artificialanalysis.ai/" }, "boardVersions": "Page text: Intelligence Index v4.3; Coding Agent Index v1.5 (same versions as round4).", "counts": { "intelligencePayloadModels": 688, "intelligenceScored": 678, "terminalBench40Rows": 206, "codingConfigurations": 31, "adoptedServedModels": 74 }, "mappingRule": "round4 slug/host mapping kept as-is; new entries are exact name matches listed in mappingAdditions. Near names stay unmapped (Qwen3.8-Flash-Next≠qwen3.8-flash, Hy3-preview≠hy3, GLM-5-Turbo≠glm-5).", "mappingAdditions": { "intelligenceSlug": { "claude-opus-5-5": "claude-opus-5.5", "claude-opus-5-5-xhigh": "claude-opus-5.5", "claude-opus-5-5-high": "claude-opus-5.5", "claude-opus-5-5-medium": "claude-opus-5.5", "claude-opus-5-5-low": "claude-opus-5.5", "claude-sonnet-5-5": "claude-sonnet-5.5", "claude-sonnet-5-5-xhigh": "claude-sonnet-5.5", "claude-sonnet-5-5-high": "claude-sonnet-5.5", "claude-sonnet-5-5-medium": "claude-sonnet-5.5", "claude-sonnet-5-5-low": "claude-sonnet-5.5", "gpt-6-1-sol": "gpt-6.1-sol", "gpt-6-1-sol-xhigh": "gpt-6.1-sol", "gpt-6-1-sol-high": "gpt-6.1-sol", "gpt-6-1-sol-medium": "gpt-6.1-sol", "gpt-6-1-sol-low": "gpt-6.1-sol", "gpt-6-sol": "gpt-6-sol", "gpt-6-sol-xhigh": "gpt-6-sol", "gpt-6-sol-high": "gpt-6-sol", "gpt-6-sol-medium": "gpt-6-sol", "gpt-6-sol-low": "gpt-6-sol", "gpt-6-sol-non-reasoning": "gpt-6-sol", "gpt-6-luna": "gpt-6-luna", "gpt-6-luna-xhigh": "gpt-6-luna", "gpt-6-luna-high": "gpt-6-luna", "gpt-6-luna-medium": "gpt-6-luna", "gpt-6-luna-low": "gpt-6-luna", "gpt-6-luna-non-reasoning": "gpt-6-luna", "grok-4-7": "grok-4.7", "grok-4-7-high": "grok-4.7", "mimo-v2-6-pro": "mimo-v2.6-pro", "mimo-v2-6-flash": "mimo-v2.6-flash", "glm-5-3-low": "glm-5.3", "deepseek-v4-1-flash-non-reasoning": "deepseek-v4.1-flash", "deepseek-v4-pro-non-reasoning": "deepseek-v4-pro", "gemini-3-6-flash": "gemini-3.6-flash", "glm-5": "glm-5", "glm-5-non-reasoning": "glm-5", "kimi-k2-5": "kimi-k2.5", "kimi-k2-5-non-reasoning": "kimi-k2.5", "mistral-medium-3-5": "mistral-medium-3.5", "qwen3-6-max": "qwen3.6-max-preview" }, "codingHost": { "anthropic_claude-opus-5-5": "claude-opus-5.5", "private-anthropic_claude-beignet-eap-max": "claude-sonnet-5.5", "private-anthropic_claude-beignet-eap-xhigh": "claude-sonnet-5.5", "private-anthropic_claude-beignet-eap-high": "claude-sonnet-5.5", "private-anthropic_claude-beignet-eap-medium": "claude-sonnet-5.5", "private-anthropic_claude-beignet-eap-low": "claude-sonnet-5.5", "openai_quasar-alpha": "gpt-6.1-sol", "openai_gpt-6-sol": "gpt-6-sol", "openai_gpt-6-luna": "gpt-6-luna", "openai_gpt-5-6-luna": "gpt-5.6-luna", "xai_grok-bop-o86-xhigh": "grok-4.7" } }, "round4SlugsAbsent": [] }, "boards": [ { "boardId": "aa_intelligence_index", "name": "Artificial Analysis Intelligence Index v4.3", "url": "https://artificialanalysis.ai/leaderboards/models", "metric": "Intelligence Index", "snapshotDate": "2026-10-01", "note": "Fresh download 2026-10-01, still Intelligence Index v4.3. Scores from public HTML RSC embedded model payload; page UI rounds to integer." }, { "boardId": "aa_coding_agent_index", "name": "Artificial Analysis Coding Agent Index v1.5", "url": "https://artificialanalysis.ai/agents/coding-agents", "metric": "Coding Agent Index", "snapshotDate": "2026-10-01", "note": "Fresh download 2026-10-01, still Coding Agent Index v1.5; 31 configurations (round4: 16)." }, { "boardId": "aa_terminal_bench_4", "name": "Terminal-Bench 4.0 (AA)", "url": "https://artificialanalysis.ai/leaderboards/models", "metric": "Resolution Rate %", "snapshotDate": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0：取 models 榜载荷 terminalBench40 字段（fraction×100），全量 206 行中按已映射模型收录。与 tbench.ai 官方榜分开计分、不可互换。" } ], "scores": [ { "model": "claude-opus-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5.5 (max with fallback)", "score": 57.6224, "secondary": { "configurationRank": 1, "costPerTaskUsd": 5.982, "scoreRoundedDisplay": 58, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-5", "shortName": "Claude Opus 5.5 (max with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "max", "terminalBench40": 0.595959595959596, "scicode": 0.668981481481482, "hle": 0.613531047265987 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5.5 (xhigh with fallback)", "score": 55.9874, "secondary": { "configurationRank": 2, "costPerTaskUsd": 3.4591, "scoreRoundedDisplay": 56, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-5-xhigh", "shortName": "Claude Opus 5.5 (xhigh with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "xhigh", "terminalBench40": 0.595959595959596, "scicode": 0.650462962962963, "hle": 0.575069508804449 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5.5 (max with fallback)", "score": 55.978, "secondary": { "configurationRank": 3, "costPerTaskUsd": 7.6206, "scoreRoundedDisplay": 56, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-5", "shortName": "Claude Sonnet 5.5 (max with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "max", "terminalBench40": 0.636363636363636, "scicode": 0.609953703703704, "hle": 0.549582947173309 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5.5 (high with fallback)", "score": 53.5832, "secondary": { "configurationRank": 4, "costPerTaskUsd": 1.8225, "scoreRoundedDisplay": 54, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-5-high", "shortName": "Claude Opus 5.5 (high with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "high", "terminalBench40": 0.565656565656566, "scicode": 0.604166666666667, "hle": 0.555607043558851 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-fable-5.1", "boardId": "aa_intelligence_index", "variantLabel": "Claude Fable 5.1 (max with fallback)", "score": 53.3549, "secondary": { "configurationRank": 5, "costPerTaskUsd": 7.6297, "scoreRoundedDisplay": 53, "intelligenceIndexIsEstimated": false, "slug": "claude-fable-5-1", "shortName": "Claude Fable 5.1 (max with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "max", "terminalBench40": 0.52020202020202, "scicode": 0.630787037037037, "hle": 0.591288229842447 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-fable-5.1", "boardId": "aa_intelligence_index", "variantLabel": "Claude Fable 5.1 (xhigh with fallback)", "score": 53.2033, "secondary": { "configurationRank": 6, "costPerTaskUsd": 5.9783, "scoreRoundedDisplay": 53, "intelligenceIndexIsEstimated": false, "slug": "claude-fable-5-1-xhigh", "shortName": "Claude Fable 5.1 (xhigh with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "xhigh", "terminalBench40": 0.55050505050505, "scicode": 0.608796296296296, "hle": 0.587117701575533 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-astra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Astra (max)", "score": 52.6737, "secondary": { "configurationRank": 7, "costPerTaskUsd": 3.2575, "scoreRoundedDisplay": 53, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-astra", "shortName": "GPT-6 Astra (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.590909090909091, "scicode": 0.564814814814815, "hle": 0.546802594995366 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-astra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Astra (xhigh)", "score": 52.3863, "secondary": { "configurationRank": 9, "costPerTaskUsd": 2.3088, "scoreRoundedDisplay": 52, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-astra-xhigh", "shortName": "GPT-6 Astra (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.595959595959596, "scicode": 0.556712962962963, "hle": 0.545875810936052 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5.5 (xhigh with fallback)", "score": 51.8523, "secondary": { "configurationRank": 10, "costPerTaskUsd": 2.7426, "scoreRoundedDisplay": 52, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-5-xhigh", "shortName": "Claude Sonnet 5.5 (xhigh with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "xhigh", "terminalBench40": 0.570707070707071, "scicode": 0.572916666666667, "hle": 0.5 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6.1-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6.1 Sol (max)", "score": 51.8333, "secondary": { "configurationRank": 11, "costPerTaskUsd": 0.7242, "scoreRoundedDisplay": 52, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-1-sol", "shortName": "GPT-6.1 Sol (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.560606060606061, "scicode": 0.541666666666667, "hle": 0.529193697868397 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5.5 (medium with fallback)", "score": 51.2435, "secondary": { "configurationRank": 12, "costPerTaskUsd": 1.336, "scoreRoundedDisplay": 51, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-5-medium", "shortName": "Claude Opus 5.5 (medium with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "medium", "terminalBench40": 0.525252525252525, "scicode": 0.592592592592593, "hle": 0.547265987025023 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-fable-5.1", "boardId": "aa_intelligence_index", "variantLabel": "Claude Fable 5.1 (high with fallback)", "score": 51.1516, "secondary": { "configurationRank": 13, "costPerTaskUsd": 3.9125, "scoreRoundedDisplay": 51, "intelligenceIndexIsEstimated": false, "slug": "claude-fable-5-1-high", "shortName": "Claude Fable 5.1 (high with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "high", "terminalBench40": 0.52020202020202, "scicode": 0.586805555555556, "hle": 0.559314179796108 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6.1-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6.1 Sol (xhigh)", "score": 51.0378, "secondary": { "configurationRank": 14, "costPerTaskUsd": 0.3929, "scoreRoundedDisplay": 51, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-1-sol-xhigh", "shortName": "GPT-6.1 Sol (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.54040404040404, "scicode": 0.556712962962963, "hle": 0.526413345690454 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-astra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Astra (high)", "score": 50.9191, "secondary": { "configurationRank": 15, "costPerTaskUsd": 1.7253, "scoreRoundedDisplay": 51, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-astra-high", "shortName": "GPT-6 Astra (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.54040404040404, "scicode": 0.554398148148148, "hle": 0.530583873957368 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5 (max)", "score": 50.7771, "secondary": { "configurationRank": 16, "costPerTaskUsd": 5.8584, "scoreRoundedDisplay": 51, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5", "shortName": "Claude Opus 5 (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "max", "terminalBench40": 0.48989898989899, "scicode": 0.563657407407407, "hle": 0.548656163113994 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6.1-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6.1 Sol (high)", "score": 50.2378, "secondary": { "configurationRank": 17, "costPerTaskUsd": 0.3191, "scoreRoundedDisplay": 50, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-1-sol-high", "shortName": "GPT-6.1 Sol (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.515151515151515, "scicode": 0.55787037037037, "hle": 0.513901760889713 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5 (xhigh)", "score": 49.6774, "secondary": { "configurationRank": 18, "costPerTaskUsd": 4.8778, "scoreRoundedDisplay": 50, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-xhigh", "shortName": "Claude Opus 5 (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "xhigh", "terminalBench40": 0.464646464646465, "scicode": 0.556712962962963, "hle": 0.544022242817424 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-fable-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Fable 5 (with fallback)", "score": 49.6258, "secondary": { "configurationRank": 19, "costPerTaskUsd": 8.746, "scoreRoundedDisplay": 50, "intelligenceIndexIsEstimated": false, "slug": "claude-fable-5", "shortName": "Claude Fable 5 (with fallback)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": null, "terminalBench40": 0.424242424242424, "scicode": 0.609953703703704, "hle": 0.554680259499537 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-astra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Astra (medium)", "score": 49.5704, "secondary": { "configurationRank": 20, "costPerTaskUsd": 1.5406, "scoreRoundedDisplay": 50, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-astra-medium", "shortName": "GPT-6 Astra (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.494949494949495, "scicode": 0.541666666666667, "hle": 0.527340129749768 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-fable-5.1", "boardId": "aa_intelligence_index", "variantLabel": "Claude Fable 5.1 (medium with fallback)", "score": 48.9206, "secondary": { "configurationRank": 21, "costPerTaskUsd": 2.9826, "scoreRoundedDisplay": 49, "intelligenceIndexIsEstimated": false, "slug": "claude-fable-5-1-medium", "shortName": "Claude Fable 5.1 (medium with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "medium", "terminalBench40": 0.44949494949495, "scicode": 0.563657407407407, "hle": 0.537998146431881 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5 (high)", "score": 48.1219, "secondary": { "configurationRank": 22, "costPerTaskUsd": 3.6133, "scoreRoundedDisplay": 48, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-high", "shortName": "Claude Opus 5 (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "high", "terminalBench40": 0.45959595959596, "scicode": 0.554398148148148, "hle": 0.528266913809083 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "muse-spark-1.3", "boardId": "aa_intelligence_index", "variantLabel": "Muse Spark 1.3 (max)", "score": 48.0923, "secondary": { "configurationRank": 23, "costPerTaskUsd": 1.6049, "scoreRoundedDisplay": 48, "intelligenceIndexIsEstimated": false, "slug": "muse-spark-1-3", "shortName": "Muse Spark 1.3 (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Meta", "effort": "max", "terminalBench40": 0.333333333333333, "scicode": 0.587962962962963, "hle": 0.487025023169602 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6.1-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6.1 Sol (medium)", "score": 47.7833, "secondary": { "configurationRank": 24, "costPerTaskUsd": 0.2137, "scoreRoundedDisplay": 48, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-1-sol-medium", "shortName": "GPT-6.1 Sol (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.47979797979798, "scicode": 0.532407407407407, "hle": 0.498609823911029 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Sol (max)", "score": 47.5276, "secondary": { "configurationRank": 25, "costPerTaskUsd": 1.0494, "scoreRoundedDisplay": 48, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-sol", "shortName": "GPT-6 Sol (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.439393939393939, "scicode": 0.576388888888889, "hle": 0.479147358665431 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Sol (max)", "score": 46.9727, "secondary": { "configurationRank": 26, "costPerTaskUsd": 1.9885, "scoreRoundedDisplay": 47, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-sol", "shortName": "GPT-5.6 Sol (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.398989898989899, "scicode": 0.570601851851852, "hle": 0.494902687673772 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-fable-5.1", "boardId": "aa_intelligence_index", "variantLabel": "Claude Fable 5.1 (low with fallback)", "score": 46.8163, "secondary": { "configurationRank": 27, "costPerTaskUsd": 2.371, "scoreRoundedDisplay": 47, "intelligenceIndexIsEstimated": false, "slug": "claude-fable-5-1-low", "shortName": "Claude Fable 5.1 (low with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "low", "terminalBench40": 0.404040404040404, "scicode": 0.56712962962963, "hle": 0.48887859128823 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5.5 (high with fallback)", "score": 46.7357, "secondary": { "configurationRank": 28, "costPerTaskUsd": 1.0798, "scoreRoundedDisplay": 47, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-5-high", "shortName": "Claude Sonnet 5.5 (high with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "high", "terminalBench40": 0.439393939393939, "scicode": 0.537037037037037, "hle": 0.458294717330862 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.7", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.7 (xhigh)", "score": 46.4466, "secondary": { "configurationRank": 29, "costPerTaskUsd": 3.7383, "scoreRoundedDisplay": 46, "intelligenceIndexIsEstimated": false, "slug": "grok-4-7", "shortName": "Grok 4.7 (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "SpaceXAI", "effort": "xhigh", "terminalBench40": 0.257575757575758, "scicode": 0.574074074074074, "hle": 0.431417979610751 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.7", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.7 (high)", "score": 46.3322, "secondary": { "configurationRank": 30, "costPerTaskUsd": 2.7261, "scoreRoundedDisplay": 46, "intelligenceIndexIsEstimated": false, "slug": "grok-4-7-high", "shortName": "Grok 4.7 (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "SpaceXAI", "effort": "high", "terminalBench40": 0.247474747474747, "scicode": 0.577546296296296, "hle": 0.423076923076923 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "mimo-v2.6-pro", "boardId": "aa_intelligence_index", "variantLabel": "MiMo-V2.6-Pro", "score": 46.3242, "secondary": { "configurationRank": 31, "costPerTaskUsd": 0.1332, "scoreRoundedDisplay": 46, "intelligenceIndexIsEstimated": false, "slug": "mimo-v2-6-pro", "shortName": "MiMo-V2.6-Pro", "isReasoning": true, "deprecated": false, "modelCreatorName": "Xiaomi", "effort": null, "terminalBench40": 0.348484848484849, "scicode": 0.608796296296296, "hle": 0.493512511584801 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-astra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Astra (low)", "score": 45.7819, "secondary": { "configurationRank": 32, "costPerTaskUsd": 0.8175, "scoreRoundedDisplay": 46, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-astra-low", "shortName": "GPT-6 Astra (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0.419191919191919, "scicode": 0.540509259259259, "hle": 0.492122335495829 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.8-max-0902", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.8 Max (0902)", "score": 45.4152, "secondary": { "configurationRank": 33, "costPerTaskUsd": 5.4085, "scoreRoundedDisplay": 45, "intelligenceIndexIsEstimated": false, "slug": "qwen3-8-max", "shortName": "Qwen3.8 Max (0902)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "effort": "max", "terminalBench40": 0.388888888888889, "scicode": 0.520833333333333, "hle": 0.430954587581094 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "muse-spark-1.3", "boardId": "aa_intelligence_index", "variantLabel": "Muse Spark 1.3 (xhigh)", "score": 45.0733, "secondary": { "configurationRank": 34, "costPerTaskUsd": 1.3678, "scoreRoundedDisplay": 45, "intelligenceIndexIsEstimated": false, "slug": "muse-spark-1-3-xhigh", "shortName": "Muse Spark 1.3 (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Meta", "effort": "xhigh", "terminalBench40": 0.166666666666667, "scicode": 0.597222222222222, "hle": 0.47451343836886 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5 (medium)", "score": 44.8253, "secondary": { "configurationRank": 35, "costPerTaskUsd": 2.1895, "scoreRoundedDisplay": 45, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-medium", "shortName": "Claude Opus 5 (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "medium", "terminalBench40": 0.343434343434343, "scicode": 0.515046296296296, "hle": 0.512974976830398 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.3", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.3 (max)", "score": 44.7774, "secondary": { "configurationRank": 36, "costPerTaskUsd": 2.0056, "scoreRoundedDisplay": 45, "intelligenceIndexIsEstimated": false, "slug": "glm-5-3", "shortName": "GLM-5.3 (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Z AI", "effort": "max", "terminalBench40": 0.419191919191919, "scicode": 0.590277777777778, "hle": 0.422613531047266 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.6", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.6 (high)", "score": 44.3113, "secondary": { "configurationRank": 37, "costPerTaskUsd": 1.8589, "scoreRoundedDisplay": 44, "intelligenceIndexIsEstimated": false, "slug": "grok-4-6", "shortName": "Grok 4.6 (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "effort": "high", "terminalBench40": 0.212121212121212, "scicode": 0.564814814814815, "hle": 0.429101019462465 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.6", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.6 (xhigh)", "score": 44.1998, "secondary": { "configurationRank": 38, "costPerTaskUsd": 2.3237, "scoreRoundedDisplay": 44, "intelligenceIndexIsEstimated": false, "slug": "grok-4-6-xhigh", "shortName": "Grok 4.6 (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "effort": "xhigh", "terminalBench40": 0.171717171717172, "scicode": 0.530092592592593, "hle": 0.440685820203893 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Sol (xhigh)", "score": 44.101, "secondary": { "configurationRank": 39, "costPerTaskUsd": 0.5236, "scoreRoundedDisplay": 44, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-sol-xhigh", "shortName": "GPT-6 Sol (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.303030303030303, "scicode": 0.550925925925926, "hle": 0.462928637627433 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Sol (xhigh)", "score": 44.0089, "secondary": { "configurationRank": 40, "costPerTaskUsd": 1.1844, "scoreRoundedDisplay": 44, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-sol-xhigh", "shortName": "GPT-5.6 Sol (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.247474747474747, "scicode": 0.570601851851852, "hle": 0.473123262279889 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "step-5-preview", "boardId": "aa_intelligence_index", "variantLabel": "Step 5 Preview", "score": 43.7343, "secondary": { "configurationRank": 41, "costPerTaskUsd": 0.7175, "scoreRoundedDisplay": 44, "intelligenceIndexIsEstimated": false, "slug": "step-5", "shortName": "Step 5 Preview", "isReasoning": true, "deprecated": false, "modelCreatorName": "StepFun", "effort": null, "terminalBench40": 0.333333333333333, "scicode": 0.58912037037037, "hle": 0.464782205746061 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "kimi-k3", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K3 (max)", "score": 43.5938, "secondary": { "configurationRank": 42, "costPerTaskUsd": 2.0001, "scoreRoundedDisplay": 44, "intelligenceIndexIsEstimated": false, "slug": "kimi-k3", "shortName": "Kimi K3 (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Kimi", "effort": "max", "terminalBench40": 0.126262626262626, "scicode": 0.594907407407407, "hle": 0.468952734012975 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.6", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.6 (medium)", "score": 42.8363, "secondary": { "configurationRank": 43, "costPerTaskUsd": 1.4963, "scoreRoundedDisplay": 43, "intelligenceIndexIsEstimated": false, "slug": "grok-4-6-medium", "shortName": "Grok 4.6 (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "effort": "medium", "terminalBench40": 0.131313131313131, "scicode": 0.559027777777778, "hle": 0.421223354958295 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Sol (high)", "score": 42.8216, "secondary": { "configurationRank": 44, "costPerTaskUsd": 0.3753, "scoreRoundedDisplay": 43, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-sol-high", "shortName": "GPT-6 Sol (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.262626262626263, "scicode": 0.548611111111111, "hle": 0.44114921223355 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Sol (high)", "score": 42.3461, "secondary": { "configurationRank": 45, "costPerTaskUsd": 0.808, "scoreRoundedDisplay": 42, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-sol-high", "shortName": "GPT-5.6 Sol (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.207070707070707, "scicode": 0.577546296296296, "hle": 0.46014828544949 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5.5 (low with fallback)", "score": 42.3078, "secondary": { "configurationRank": 46, "costPerTaskUsd": 0.5512, "scoreRoundedDisplay": 42, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-5-low", "shortName": "Claude Opus 5.5 (low with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "low", "terminalBench40": 0.313131313131313, "scicode": 0.585648148148148, "hle": 0.483317886932345 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6.1-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6.1 Sol (low)", "score": 42.0836, "secondary": { "configurationRank": 47, "costPerTaskUsd": 0.1308, "scoreRoundedDisplay": 42, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-1-sol-low", "shortName": "GPT-6.1 Sol (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0.308080808080808, "scicode": 0.532407407407407, "hle": 0.473586654309546 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-terra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Terra (max)", "score": 42.0829, "secondary": { "configurationRank": 48, "costPerTaskUsd": 1.3987, "scoreRoundedDisplay": 42, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-terra", "shortName": "GPT-5.6 Terra (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.353535353535354, "scicode": 0.549768518518518, "hle": 0.429101019462465 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.3-flash", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.3-Flash", "score": 41.8075, "secondary": { "configurationRank": 49, "costPerTaskUsd": 0.2533, "scoreRoundedDisplay": 42, "intelligenceIndexIsEstimated": false, "slug": "glm-5-3-flash", "shortName": "GLM-5.3-Flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "Z AI", "effort": null, "terminalBench40": 0.328282828282828, "scicode": 0.516203703703704, "hle": 0.398517145505097 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-4.8", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 4.8 (max)", "score": 41.7899, "secondary": { "configurationRank": 50, "costPerTaskUsd": 4.081, "scoreRoundedDisplay": 42, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-4-8", "shortName": "Claude Opus 4.8 (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "max", "terminalBench40": 0.217171717171717, "scicode": 0.543981481481482, "hle": 0.486561631139944 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.8-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.8 Flash (high)", "score": 40.9262, "secondary": { "configurationRank": 51, "costPerTaskUsd": 1.2428, "scoreRoundedDisplay": 41, "intelligenceIndexIsEstimated": false, "slug": "gemini-3-8-flash", "shortName": "Gemini 3.8 Flash (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "effort": "high", "terminalBench40": 0.196969696969697, "scicode": 0.565972222222222, "hle": 0.478220574606117 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5.5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5.5 (medium with fallback)", "score": 40.7387, "secondary": { "configurationRank": 52, "costPerTaskUsd": 0.5862, "scoreRoundedDisplay": 41, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-5-medium", "shortName": "Claude Sonnet 5.5 (medium with fallback)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "effort": "medium", "terminalBench40": 0.297979797979798, "scicode": 0.528935185185185, "hle": 0.39805375347544 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.8-max", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.8 Max", "score": 40.1529, "secondary": { "configurationRank": 54, "costPerTaskUsd": 2.67, "scoreRoundedDisplay": 40, "intelligenceIndexIsEstimated": false, "slug": "qwen3-8-max-0803", "shortName": "Qwen3.8 Max", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "effort": null, "terminalBench40": 0.186868686868687, "scicode": 0.532407407407407, "hle": 0.430491195551437 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Sol (medium)", "score": 39.7821, "secondary": { "configurationRank": 57, "costPerTaskUsd": 0.2472, "scoreRoundedDisplay": 40, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-sol-medium", "shortName": "GPT-6 Sol (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.186868686868687, "scicode": 0.538194444444444, "hle": 0.409638554216867 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.8-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.8 Flash (medium)", "score": 39.774, "secondary": { "configurationRank": 58, "costPerTaskUsd": 0.931, "scoreRoundedDisplay": 40, "intelligenceIndexIsEstimated": false, "slug": "gemini-3-8-flash-medium", "shortName": "Gemini 3.8 Flash (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "effort": "medium", "terminalBench40": 0.196969696969697, "scicode": 0.550925925925926, "hle": 0.421223354958295 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.7-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.7 Flash (medium)", "score": 39.6178, "secondary": { "configurationRank": 59, "costPerTaskUsd": null, "scoreRoundedDisplay": 40, "intelligenceIndexIsEstimated": true, "slug": "gemini-3-7-flash-medium", "shortName": "Gemini 3.7 Flash (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "effort": "medium", "terminalBench40": null, "scicode": 0.59837962962963, "hle": 0.389712696941613 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "muse-spark-1.2", "boardId": "aa_intelligence_index", "variantLabel": "Muse Spark 1.2 (xhigh)", "score": 39.5759, "secondary": { "configurationRank": 60, "costPerTaskUsd": 0.9747, "scoreRoundedDisplay": 40, "intelligenceIndexIsEstimated": false, "slug": "muse-spark-1-2", "shortName": "Muse Spark 1.2 (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Meta", "effort": "xhigh", "terminalBench40": 0.0707070707070707, "scicode": 0.574074074074074, "hle": 0.454587581093605 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4.1-flash", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4.1 Flash (max)", "score": 39.4562, "secondary": { "configurationRank": 61, "costPerTaskUsd": 0.2652, "scoreRoundedDisplay": 39, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-1-flash", "shortName": "DeepSeek V4.1 Flash (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "DeepSeek", "effort": "max", "terminalBench40": 0.267676767676768, "scicode": 0.518518518518518, "hle": 0.392493049119555 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-opus-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Opus 5 (low)", "score": 39.3543, "secondary": { "configurationRank": 62, "costPerTaskUsd": 1.0983, "scoreRoundedDisplay": 39, "intelligenceIndexIsEstimated": false, "slug": "claude-opus-5-low", "shortName": "Claude Opus 5 (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "low", "terminalBench40": 0.262626262626263, "scicode": 0.491898148148148, "hle": 0.434198331788693 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Sol (medium)", "score": 39.2364, "secondary": { "configurationRank": 63, "costPerTaskUsd": 0.505, "scoreRoundedDisplay": 39, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-sol-medium", "shortName": "GPT-5.6 Sol (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.146464646464646, "scicode": 0.574074074074074, "hle": 0.422150139017609 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.7-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.7 Flash (high)", "score": 39.0595, "secondary": { "configurationRank": 64, "costPerTaskUsd": 0.9253, "scoreRoundedDisplay": 39, "intelligenceIndexIsEstimated": false, "slug": "gemini-3-7-flash", "shortName": "Gemini 3.7 Flash (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "effort": "high", "terminalBench40": 0.136363636363636, "scicode": 0.571759259259259, "hle": 0.478683966635774 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.5", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.5 (high)", "score": 38.8121, "secondary": { "configurationRank": 66, "costPerTaskUsd": 1.0365, "scoreRoundedDisplay": 39, "intelligenceIndexIsEstimated": false, "slug": "grok-4-5", "shortName": "Grok 4.5 (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "effort": "high", "terminalBench40": 0.106060606060606, "scicode": 0.549768518518518, "hle": 0.42678405931418 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.5", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.5 (xhigh)", "score": 38.3556, "secondary": { "configurationRank": 67, "costPerTaskUsd": 2.634, "scoreRoundedDisplay": 38, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-5", "shortName": "GPT-5.5 (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.146464646464646, "scicode": 0.55787037037037, "hle": 0.457831325301205 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5 (max)", "score": 38.1639, "secondary": { "configurationRank": 68, "costPerTaskUsd": 5.0912, "scoreRoundedDisplay": 38, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5", "shortName": "Claude Sonnet 5 (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "max", "terminalBench40": 0.141414141414141, "scicode": 0.542824074074074, "hle": 0.412882298424467 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-terra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Terra (xhigh)", "score": 37.9513, "secondary": { "configurationRank": 69, "costPerTaskUsd": 0.6318, "scoreRoundedDisplay": 38, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-terra-xhigh", "shortName": "GPT-5.6 Terra (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.101010101010101, "scicode": 0.523148148148148, "hle": 0.418906394810009 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "mimo-v2.6-flash", "boardId": "aa_intelligence_index", "variantLabel": "MiMo-V2.6-Flash", "score": 37.8844, "secondary": { "configurationRank": 70, "costPerTaskUsd": 0.0622, "scoreRoundedDisplay": 38, "intelligenceIndexIsEstimated": false, "slug": "mimo-v2-6-flash", "shortName": "MiMo-V2.6-Flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "Xiaomi", "effort": null, "terminalBench40": 0.227272727272727, "scicode": 0.512731481481482, "hle": 0.350787766450417 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Luna (max)", "score": 37.3244, "secondary": { "configurationRank": 71, "costPerTaskUsd": 0.1783, "scoreRoundedDisplay": 37, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-luna", "shortName": "GPT-5.6 Luna (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.116161616161616, "scicode": 0.53587962962963, "hle": 0.394810009267841 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Luna (max)", "score": 37.256, "secondary": { "configurationRank": 72, "costPerTaskUsd": 0.0679, "scoreRoundedDisplay": 37, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-luna", "shortName": "GPT-6 Luna (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "max", "terminalBench40": 0.126262626262626, "scicode": 0.546296296296296, "hle": 0.385078776645042 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.5", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.5 (high)", "score": 36.9794, "secondary": { "configurationRank": 73, "costPerTaskUsd": 1.5413, "scoreRoundedDisplay": 37, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-5-high", "shortName": "GPT-5.5 (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.0909090909090909, "scicode": 0.561342592592593, "hle": 0.450417052826691 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.7-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.7 Flash (low)", "score": 36.9459, "secondary": { "configurationRank": 74, "costPerTaskUsd": null, "scoreRoundedDisplay": 37, "intelligenceIndexIsEstimated": true, "slug": "gemini-3-7-flash-low", "shortName": "Gemini 3.7 Flash (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "effort": "low", "terminalBench40": null, "scicode": 0.556712962962963, "hle": 0.351251158480074 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "deepseek-v4-pro", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Pro 0813 (max)", "score": 35.9968, "secondary": { "configurationRank": 75, "costPerTaskUsd": 0.674, "scoreRoundedDisplay": 36, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-pro", "shortName": "DeepSeek V4 Pro 0813 (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "DeepSeek", "effort": "max", "terminalBench40": 0.141414141414141, "scicode": 0.510416666666667, "hle": 0.410101946246525 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "grok-4.6", "boardId": "aa_intelligence_index", "variantLabel": "Grok 4.6 (low)", "score": 35.1247, "secondary": { "configurationRank": 76, "costPerTaskUsd": 0.4753, "scoreRoundedDisplay": 35, "intelligenceIndexIsEstimated": false, "slug": "grok-4-6-low", "shortName": "Grok 4.6 (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "effort": "low", "terminalBench40": 0.0303030303030303, "scicode": 0.494212962962963, "hle": 0.276181649675626 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-flash-vision-exp", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Flash Vision (max)", "score": 34.8391, "secondary": { "configurationRank": 77, "costPerTaskUsd": 0.3144, "scoreRoundedDisplay": 35, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-flash-vision", "shortName": "DeepSeek V4 Flash Vision (max)", "isReasoning": true, "deprecated": false, "modelCreatorName": "DeepSeek", "effort": "max", "terminalBench40": 0.121212121212121, "scicode": 0.496527777777778, "hle": 0.344763670064875 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Luna (xhigh)", "score": 34.5553, "secondary": { "configurationRank": 78, "costPerTaskUsd": 0.0853, "scoreRoundedDisplay": 35, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-luna-xhigh", "shortName": "GPT-5.6 Luna (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.0353535353535354, "scicode": 0.50462962962963, "hle": 0.369786839666358 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5 (xhigh)", "score": 34.3842, "secondary": { "configurationRank": 79, "costPerTaskUsd": 2.872, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-xhigh", "shortName": "Claude Sonnet 5 (xhigh)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "xhigh", "terminalBench40": 0.0707070707070707, "scicode": 0.540509259259259, "hle": 0.39017608897127 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-flash", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Flash 0731 (max)", "score": 34.3305, "secondary": { "configurationRank": 80, "costPerTaskUsd": 0.2196, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-flash", "shortName": "DeepSeek V4 Flash 0731 (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "max", "terminalBench40": 0.121212121212121, "scicode": 0.503472222222222, "hle": 0.385542168674699 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.3", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.3 (low)", "score": 34.299, "secondary": { "configurationRank": 81, "costPerTaskUsd": 0.8522, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "glm-5-3-low", "shortName": "GLM-5.3 (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Z AI", "effort": "low", "terminalBench40": 0.348484848484849, "scicode": 0.420138888888889, "hle": 0.365616311399444 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-terra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Terra (high)", "score": 34.2377, "secondary": { "configurationRank": 82, "costPerTaskUsd": 0.3379, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-terra-high", "shortName": "GPT-5.6 Terra (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.0151515151515152, "scicode": 0.524305555555556, "hle": 0.385078776645042 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.6-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.6 Flash (high)", "score": 33.9786, "secondary": { "configurationRank": 83, "costPerTaskUsd": 0.9288, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "gemini-3-6-flash", "shortName": "Gemini 3.6 Flash (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "effort": "high", "terminalBench40": 0.0707070707070707, "scicode": 0.533564814814815, "hle": 0.408248378127896 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Sol (low)", "score": 33.9009, "secondary": { "configurationRank": 85, "costPerTaskUsd": 0.1326, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-sol-low", "shortName": "GPT-6 Sol (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0.0909090909090909, "scicode": 0.502314814814815, "hle": 0.349397590361446 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Luna (xhigh)", "score": 33.8846, "secondary": { "configurationRank": 86, "costPerTaskUsd": 0.0421, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-luna-xhigh", "shortName": "GPT-6 Luna (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "xhigh", "terminalBench40": 0.0808080808080808, "scicode": 0.517361111111111, "hle": 0.343373493975904 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.5", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.5 (medium)", "score": 33.8044, "secondary": { "configurationRank": 87, "costPerTaskUsd": 0.9017, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-5-medium", "shortName": "GPT-5.5 (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.0505050505050505, "scicode": 0.545138888888889, "hle": 0.42354031510658 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.2", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.2 (max)", "score": 33.7055, "secondary": { "configurationRank": 89, "costPerTaskUsd": 1.4726, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "glm-5-2", "shortName": "GLM-5.2 (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "effort": "max", "terminalBench40": 0.0101010101010101, "scicode": 0.511574074074074, "hle": 0.411492122335496 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.8-27b", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.8 27B (xhigh)", "score": 33.6963, "secondary": { "configurationRank": 90, "costPerTaskUsd": 1.0073, "scoreRoundedDisplay": 34, "intelligenceIndexIsEstimated": false, "slug": "qwen3-8-27b", "shortName": "Qwen3.8 27B (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "effort": "xhigh", "terminalBench40": 0.0555555555555556, "scicode": 0.466435185185185, "hle": 0.33920296570899 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Sol (low)", "score": 33.4731, "secondary": { "configurationRank": 93, "costPerTaskUsd": 0.2606, "scoreRoundedDisplay": 33, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-sol-low", "shortName": "GPT-5.6 Sol (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0.0101010101010101, "scicode": 0.563657407407407, "hle": 0.393883225208526 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.8-flash", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.8 Flash (low)", "score": 33.4548, "secondary": { "configurationRank": 94, "costPerTaskUsd": null, "scoreRoundedDisplay": 33, "intelligenceIndexIsEstimated": false, "slug": "gemini-3-8-flash-low", "shortName": "Gemini 3.8 Flash (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "effort": "low", "terminalBench40": 0.101010101010101, "scicode": 0.549768518518518, "hle": 0.370713623725672 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Luna (high)", "score": 32.1482, "secondary": { "configurationRank": 98, "costPerTaskUsd": 0.0291, "scoreRoundedDisplay": 32, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-luna-high", "shortName": "GPT-6 Luna (high)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.0454545454545455, "scicode": 0.503472222222222, "hle": 0.329471733086191 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Luna (high)", "score": 32.1196, "secondary": { "configurationRank": 99, "costPerTaskUsd": 0.044, "scoreRoundedDisplay": 32, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-luna-high", "shortName": "GPT-5.6 Luna (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "high", "terminalBench40": 0.0252525252525253, "scicode": 0.516203703703704, "hle": 0.334105653382762 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5 (high)", "score": 31.6623, "secondary": { "configurationRank": 101, "costPerTaskUsd": 1.7925, "scoreRoundedDisplay": 32, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-high", "shortName": "Claude Sonnet 5 (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "high", "terminalBench40": 0.0505050505050505, "scicode": 0.542824074074074, "hle": 0.357275254865616 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.5", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.5 (low)", "score": 30.7095, "secondary": { "configurationRank": 104, "costPerTaskUsd": null, "scoreRoundedDisplay": 31, "intelligenceIndexIsEstimated": true, "slug": "gpt-5-5-low", "shortName": "GPT-5.5 (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": null, "scicode": null, "hle": 0.326691380908248 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "deepseek-v4-pro", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Pro (max)", "score": 30.4497, "secondary": { "configurationRank": 106, "costPerTaskUsd": 0.1215, "scoreRoundedDisplay": 30, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-pro-0424", "shortName": "DeepSeek V4 Pro (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "max", "terminalBench40": 0.146464646464646, "scicode": 0.508101851851852, "hle": 0.375347544022243 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-pro", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Pro (high)", "score": 30.1055, "secondary": { "configurationRank": 108, "costPerTaskUsd": null, "scoreRoundedDisplay": 30, "intelligenceIndexIsEstimated": true, "slug": "deepseek-v4-pro-0424-high", "shortName": "DeepSeek V4 Pro (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "high", "terminalBench40": null, "scicode": null, "hle": 0.352177942539388 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-5.6-terra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Terra (medium)", "score": 30.0938, "secondary": { "configurationRank": 109, "costPerTaskUsd": 0.1834, "scoreRoundedDisplay": 30, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-terra-medium", "shortName": "GPT-5.6 Terra (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.0101010101010101, "scicode": 0.50462962962963, "hle": 0.332715477293791 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "kimi-k3", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K3 (low)", "score": 30.0667, "secondary": { "configurationRank": 110, "costPerTaskUsd": 1.1463, "scoreRoundedDisplay": 30, "intelligenceIndexIsEstimated": false, "slug": "kimi-k3-low", "shortName": "Kimi K3 (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Kimi", "effort": "low", "terminalBench40": 0.126262626262626, "scicode": 0.52662037037037, "hle": 0.249768303985171 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gemini-3.1-pro", "boardId": "aa_intelligence_index", "variantLabel": "Gemini 3.1 Pro Preview", "score": 29.7186, "secondary": { "configurationRank": 112, "costPerTaskUsd": 0.6747, "scoreRoundedDisplay": 30, "intelligenceIndexIsEstimated": false, "slug": "gemini-3-1-pro-preview", "shortName": "Gemini 3.1 Pro Preview", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "effort": null, "terminalBench40": 0.0404040404040404, "scicode": 0.586805555555556, "hle": 0.470342910101946 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Luna (medium)", "score": 29.462, "secondary": { "configurationRank": 113, "costPerTaskUsd": 0.0176, "scoreRoundedDisplay": 29, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-luna-medium", "shortName": "GPT-6 Luna (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.0252525252525253, "scicode": 0.509259259259259, "hle": 0.282669138090825 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.7-max", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.7 Max", "score": 29.4572, "secondary": { "configurationRank": 114, "costPerTaskUsd": 1.1473, "scoreRoundedDisplay": 29, "intelligenceIndexIsEstimated": false, "slug": "qwen3-7-max", "shortName": "Qwen3.7 Max", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "effort": "max", "terminalBench40": 0.0151515151515152, "scicode": 0.49537037037037, "hle": 0.405004633920297 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "minimax-m3", "boardId": "aa_intelligence_index", "variantLabel": "MiniMax-M3", "score": 29.2203, "secondary": { "configurationRank": 115, "costPerTaskUsd": 0.5076, "scoreRoundedDisplay": 29, "intelligenceIndexIsEstimated": false, "slug": "minimax-m3", "shortName": "MiniMax-M3", "isReasoning": true, "deprecated": false, "modelCreatorName": "MiniMax", "effort": null, "terminalBench40": 0.0202020202020202, "scicode": 0.471064814814815, "hle": 0.389712696941613 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.6-max-preview", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.6 Max Preview", "score": 28.3749, "secondary": { "configurationRank": 119, "costPerTaskUsd": null, "scoreRoundedDisplay": 28, "intelligenceIndexIsEstimated": true, "slug": "qwen3-6-max", "shortName": "Qwen3.6 Max Preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "effort": "max", "terminalBench40": null, "scicode": null, "hle": 0.308155699721965 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-5.6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Sol (non-reasoning)", "score": 28.328, "secondary": { "configurationRank": 120, "costPerTaskUsd": null, "scoreRoundedDisplay": 28, "intelligenceIndexIsEstimated": true, "slug": "gpt-5-6-sol-non-reasoning", "shortName": "GPT-5.6 Sol (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "non-reasoning", "terminalBench40": null, "scicode": 0.476851851851852, "hle": 0.166821130676552 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-6-sol", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Sol (non-reasoning)", "score": 28.0922, "secondary": { "configurationRank": 123, "costPerTaskUsd": 0.3334, "scoreRoundedDisplay": 28, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-sol-non-reasoning", "shortName": "GPT-6 Sol (non-reasoning)", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "non-reasoning", "terminalBench40": 0.131313131313131, "scicode": 0.47337962962963, "hle": 0.183966635773865 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5 (medium)", "score": 28.0531, "secondary": { "configurationRank": 124, "costPerTaskUsd": 0.9991, "scoreRoundedDisplay": 28, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-medium", "shortName": "Claude Sonnet 5 (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "medium", "terminalBench40": 0.0202020202020202, "scicode": 0.516203703703704, "hle": 0.299814643188137 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5", "score": 27.9112, "secondary": { "configurationRank": 126, "costPerTaskUsd": null, "scoreRoundedDisplay": 28, "intelligenceIndexIsEstimated": true, "slug": "glm-5", "shortName": "GLM-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "effort": null, "terminalBench40": null, "scicode": null, "hle": 0.292863762743281 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "qwen3.8-27b", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.8 27B (medium)", "score": 27.5508, "secondary": { "configurationRank": 128, "costPerTaskUsd": 1.1331, "scoreRoundedDisplay": 28, "intelligenceIndexIsEstimated": false, "slug": "qwen3-8-27b-medium", "shortName": "Qwen3.8 27B (medium)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "effort": "medium", "terminalBench40": 0.0505050505050505, "scicode": 0.390046296296296, "hle": 0.140871177015755 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-terra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Terra (low)", "score": 27.4962, "secondary": { "configurationRank": 129, "costPerTaskUsd": 0.1445, "scoreRoundedDisplay": 27, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-terra-low", "shortName": "GPT-5.6 Terra (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0.0151515151515152, "scicode": 0.498842592592593, "hle": 0.291936978683967 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.6-plus", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.6 Plus", "score": 27.0099, "secondary": { "configurationRank": 132, "costPerTaskUsd": null, "scoreRoundedDisplay": 27, "intelligenceIndexIsEstimated": true, "slug": "qwen3-6-plus", "shortName": "Qwen3.6 Plus", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "effort": null, "terminalBench40": null, "scicode": null, "hle": 0.278498609823911 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "kimi-k2.6", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K2.6", "score": 26.9792, "secondary": { "configurationRank": 133, "costPerTaskUsd": 0.7998, "scoreRoundedDisplay": 27, "intelligenceIndexIsEstimated": false, "slug": "kimi-k2-6", "shortName": "Kimi K2.6", "isReasoning": true, "deprecated": true, "modelCreatorName": "Kimi", "effort": null, "terminalBench40": 0.00505050505050505, "scicode": 0.515046296296296, "hle": 0.374884151992586 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.8-27b", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.8 27B (low)", "score": 26.2048, "secondary": { "configurationRank": 140, "costPerTaskUsd": 1.0482, "scoreRoundedDisplay": 26, "intelligenceIndexIsEstimated": false, "slug": "qwen3-8-27b-low", "shortName": "Qwen3.8 27B (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "effort": "low", "terminalBench40": 0.0252525252525253, "scicode": 0.400462962962963, "hle": 0.140407784986098 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.1", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.1", "score": 26.0586, "secondary": { "configurationRank": 141, "costPerTaskUsd": 1.0193, "scoreRoundedDisplay": 26, "intelligenceIndexIsEstimated": false, "slug": "glm-5-1", "shortName": "GLM-5.1", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "effort": null, "terminalBench40": 0.0202020202020202, "scicode": 0.447916666666667, "hle": 0.300741427247451 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "mimo-v2.5-pro", "boardId": "aa_intelligence_index", "variantLabel": "MiMo-V2.5-Pro", "score": 25.9869, "secondary": { "configurationRank": 143, "costPerTaskUsd": 0.0541, "scoreRoundedDisplay": 26, "intelligenceIndexIsEstimated": false, "slug": "mimo-v2-5-pro", "shortName": "MiMo-V2.5-Pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "effort": null, "terminalBench40": 0, "scicode": 0.505787037037037, "hle": 0.356811862835959 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "kimi-k2.7-code", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K2.7 Code", "score": 25.8121, "secondary": { "configurationRank": 144, "costPerTaskUsd": 0.5412, "scoreRoundedDisplay": 26, "intelligenceIndexIsEstimated": false, "slug": "kimi-k2-7-code", "shortName": "Kimi K2.7 Code", "isReasoning": true, "deprecated": false, "modelCreatorName": "Kimi", "effort": null, "terminalBench40": 0.0101010101010101, "scicode": 0.478009259259259, "hle": 0.35032437442076 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "inkling-small", "boardId": "aa_intelligence_index", "variantLabel": "Inkling Small", "score": 25.6583, "secondary": { "configurationRank": 145, "costPerTaskUsd": 0.0895, "scoreRoundedDisplay": 26, "intelligenceIndexIsEstimated": false, "slug": "inkling-small", "shortName": "Inkling Small", "isReasoning": true, "deprecated": false, "modelCreatorName": "Thinking Machines", "effort": null, "terminalBench40": 0.0101010101010101, "scicode": 0.496527777777778, "hle": 0.333178869323448 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "hy3", "boardId": "aa_intelligence_index", "variantLabel": "Hy3", "score": 25.2973, "secondary": { "configurationRank": 148, "costPerTaskUsd": 0.0718, "scoreRoundedDisplay": 25, "intelligenceIndexIsEstimated": false, "slug": "hy3", "shortName": "Hy3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Tencent", "effort": null, "terminalBench40": 0.00505050505050505, "scicode": 0.486111111111111, "hle": 0.334569045412419 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "mimo-v2.5", "boardId": "aa_intelligence_index", "variantLabel": "MiMo-V2.5", "score": 25.1743, "secondary": { "configurationRank": 150, "costPerTaskUsd": null, "scoreRoundedDisplay": 25, "intelligenceIndexIsEstimated": true, "slug": "mimo-v2-5-0424", "shortName": "MiMo-V2.5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "effort": null, "terminalBench40": 0, "scicode": 0.438657407407407, "hle": 0.272011121408712 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "qwen3.7-plus", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.7 Plus", "score": 25.1622, "secondary": { "configurationRank": 151, "costPerTaskUsd": 0.217, "scoreRoundedDisplay": 25, "intelligenceIndexIsEstimated": false, "slug": "qwen3-7-plus", "shortName": "Qwen3.7 Plus", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "effort": null, "terminalBench40": 0.0101010101010101, "scicode": 0.460648148148148, "hle": 0.356348470806302 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-5.6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Luna (medium)", "score": 25.0359, "secondary": { "configurationRank": 153, "costPerTaskUsd": 0.0156, "scoreRoundedDisplay": 25, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-luna-medium", "shortName": "GPT-5.6 Luna (medium)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "medium", "terminalBench40": 0.00505050505050505, "scicode": 0.467592592592593, "hle": 0.257645968489342 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "inkling", "boardId": "aa_intelligence_index", "variantLabel": "Inkling (xhigh)", "score": 24.9848, "secondary": { "configurationRank": 154, "costPerTaskUsd": null, "scoreRoundedDisplay": 25, "intelligenceIndexIsEstimated": false, "slug": "inkling", "shortName": "Inkling (xhigh)", "isReasoning": true, "deprecated": false, "modelCreatorName": "Thinking Machines", "effort": "xhigh", "terminalBench40": 0.0101010101010101, "scicode": 0.469907407407407, "hle": 0.318813716404078 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4.1-flash", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4.1 Flash (non-reasoning)", "score": 24.6734, "secondary": { "configurationRank": 161, "costPerTaskUsd": 0.1467, "scoreRoundedDisplay": 25, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-1-flash-non-reasoning", "shortName": "DeepSeek V4.1 Flash (non-reasoning)", "isReasoning": false, "deprecated": false, "modelCreatorName": "DeepSeek", "effort": "non-reasoning", "terminalBench40": 0.0555555555555556, "scicode": 0.355324074074074, "hle": 0.107970342910102 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-flash", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Flash (high)", "score": 24.384, "secondary": { "configurationRank": 163, "costPerTaskUsd": 0.1271, "scoreRoundedDisplay": 24, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-flash-0420-high", "shortName": "DeepSeek V4 Flash (high)", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "high", "terminalBench40": 0.0303030303030303, "scicode": 0.40162037037037, "hle": 0.30259499536608 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5 (low)", "score": 24.2638, "secondary": { "configurationRank": 165, "costPerTaskUsd": 0.5088, "scoreRoundedDisplay": 24, "intelligenceIndexIsEstimated": false, "slug": "claude-sonnet-5-low", "shortName": "Claude Sonnet 5 (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "low", "terminalBench40": 0.0252525252525253, "scicode": 0.501157407407407, "hle": 0.219184430027804 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.1", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.1 (non-reasoning)", "score": 24.2412, "secondary": { "configurationRank": 166, "costPerTaskUsd": null, "scoreRoundedDisplay": 24, "intelligenceIndexIsEstimated": true, "slug": "glm-5-1-non-reasoning", "shortName": "GLM-5.1 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.279425393883225 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "deepseek-v4-flash", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Flash (max)", "score": 24.1673, "secondary": { "configurationRank": 167, "costPerTaskUsd": 0.1075, "scoreRoundedDisplay": 24, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-flash-0420", "shortName": "DeepSeek V4 Flash (max)", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "max", "terminalBench40": 0.0252525252525253, "scicode": 0.452546296296296, "hle": 0.348470806302132 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "kimi-k2.6", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K2.6 (non-reasoning)", "score": 23.5696, "secondary": { "configurationRank": 174, "costPerTaskUsd": null, "scoreRoundedDisplay": 24, "intelligenceIndexIsEstimated": true, "slug": "kimi-k2-6-non-reasoning", "shortName": "Kimi K2.6 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Kimi", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.195551436515292 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "kimi-k2.5", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K2.5", "score": 23.4585, "secondary": { "configurationRank": 176, "costPerTaskUsd": null, "scoreRoundedDisplay": 23, "intelligenceIndexIsEstimated": true, "slug": "kimi-k2-5", "shortName": "Kimi K2.5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Kimi", "effort": null, "terminalBench40": null, "scicode": null, "hle": 0.307228915662651 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "claude-sonnet-5", "boardId": "aa_intelligence_index", "variantLabel": "Claude Sonnet 5 (non-reasoning)", "score": 23.2, "secondary": { "configurationRank": 178, "costPerTaskUsd": null, "scoreRoundedDisplay": 23, "intelligenceIndexIsEstimated": true, "slug": "claude-sonnet-5-non-reasoning", "shortName": "Claude Sonnet 5 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.190454124189064 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-5.5", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.5 (non-reasoning)", "score": 23.1674, "secondary": { "configurationRank": 179, "costPerTaskUsd": null, "scoreRoundedDisplay": 23, "intelligenceIndexIsEstimated": true, "slug": "gpt-5-5-non-reasoning", "shortName": "GPT-5.5 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.137164040778499 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "nemotron-3-ultra", "boardId": "aa_intelligence_index", "variantLabel": "Nemotron 3 Ultra", "score": 22.9279, "secondary": { "configurationRank": 181, "costPerTaskUsd": 0.5976, "scoreRoundedDisplay": 23, "intelligenceIndexIsEstimated": false, "slug": "nvidia-nemotron-3-ultra-550b-a55b", "shortName": "Nemotron 3 Ultra", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "effort": null, "terminalBench40": 0.00505050505050505, "scicode": 0.402777777777778, "hle": 0.284059314179796 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "minimax-m2.5", "boardId": "aa_intelligence_index", "variantLabel": "MiniMax-M2.5", "score": 22.7966, "secondary": { "configurationRank": 185, "costPerTaskUsd": null, "scoreRoundedDisplay": 23, "intelligenceIndexIsEstimated": true, "slug": "minimax-m2-5", "shortName": "MiniMax-M2.5", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "effort": null, "terminalBench40": null, "scicode": null, "hle": 0.205282669138091 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "minimax-m2.7", "boardId": "aa_intelligence_index", "variantLabel": "MiniMax-M2.7", "score": 22.7578, "secondary": { "configurationRank": 186, "costPerTaskUsd": null, "scoreRoundedDisplay": 23, "intelligenceIndexIsEstimated": false, "slug": "minimax-m2-7", "shortName": "MiniMax-M2.7", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "effort": null, "terminalBench40": 0, "scicode": 0.501157407407407, "hle": 0.29610750695088 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "glm-5.2", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5.2 (non-reasoning)", "score": 22.4323, "secondary": { "configurationRank": 193, "costPerTaskUsd": null, "scoreRoundedDisplay": 22, "intelligenceIndexIsEstimated": true, "slug": "glm-5-2-non-reasoning", "shortName": "GLM-5.2 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.097775718257646 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "glm-5", "boardId": "aa_intelligence_index", "variantLabel": "GLM-5 (non-reasoning)", "score": 21.7826, "secondary": { "configurationRank": 200, "costPerTaskUsd": null, "scoreRoundedDisplay": 22, "intelligenceIndexIsEstimated": true, "slug": "glm-5-non-reasoning", "shortName": "GLM-5 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.0764596848934198 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-5.6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Luna (low)", "score": 21.0125, "secondary": { "configurationRank": 206, "costPerTaskUsd": 0.0098, "scoreRoundedDisplay": 21, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-luna-low", "shortName": "GPT-5.6 Luna (low)", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0, "scicode": 0.460648148148148, "hle": 0.198331788693234 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "gpt-6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Luna (low)", "score": 20.9225, "secondary": { "configurationRank": 208, "costPerTaskUsd": 0.0045, "scoreRoundedDisplay": 21, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-luna-low", "shortName": "GPT-6 Luna (low)", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "low", "terminalBench40": 0, "scicode": 0.46875, "hle": 0.202502316960148 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-pro", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Pro (non-reasoning)", "score": 20.8345, "secondary": { "configurationRank": 209, "costPerTaskUsd": null, "scoreRoundedDisplay": 21, "intelligenceIndexIsEstimated": true, "slug": "deepseek-v4-pro-0424-non-reasoning", "shortName": "DeepSeek V4 Pro (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.082483781278962 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-5.6-terra", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Terra (non-reasoning)", "score": 20.7797, "secondary": { "configurationRank": 212, "costPerTaskUsd": 0.1397, "scoreRoundedDisplay": 21, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-terra-non-reasoning", "shortName": "GPT-5.6 Terra (non-reasoning)", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "non-reasoning", "terminalBench40": 0.00505050505050505, "scicode": 0.451388888888889, "hle": 0.113994439295644 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-pro", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Pro 0813 (non-reasoning)", "score": 20.3605, "secondary": { "configurationRank": 221, "costPerTaskUsd": 0.4774, "scoreRoundedDisplay": 20, "intelligenceIndexIsEstimated": false, "slug": "deepseek-v4-pro-non-reasoning", "shortName": "DeepSeek V4 Pro 0813 (non-reasoning)", "isReasoning": false, "deprecated": false, "modelCreatorName": "DeepSeek", "effort": "non-reasoning", "terminalBench40": 0.0151515151515152, "scicode": 0.399305555555556, "hle": 0.105653382761817 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "qwen3.8-27b", "boardId": "aa_intelligence_index", "variantLabel": "Qwen3.8 27B (non-reasoning)", "score": 20.1502, "secondary": { "configurationRank": 223, "costPerTaskUsd": 2.4878, "scoreRoundedDisplay": 20, "intelligenceIndexIsEstimated": false, "slug": "qwen3-8-27b-non-reasoning", "shortName": "Qwen3.8 27B (non-reasoning)", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "effort": "non-reasoning", "terminalBench40": 0, "scicode": 0.362268518518519, "hle": 0.121408711770158 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "step-3.7-flash", "boardId": "aa_intelligence_index", "variantLabel": "Step 3.7 Flash", "score": 19.4813, "secondary": { "configurationRank": 229, "costPerTaskUsd": null, "scoreRoundedDisplay": 19, "intelligenceIndexIsEstimated": true, "slug": "step-3-7-flash", "shortName": "Step 3.7 Flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "StepFun", "effort": null, "terminalBench40": null, "scicode": 0.438657407407407, "hle": 0.214087117701576 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "kimi-k2.5", "boardId": "aa_intelligence_index", "variantLabel": "Kimi K2.5 (non-reasoning)", "score": 19.4343, "secondary": { "configurationRank": 230, "costPerTaskUsd": null, "scoreRoundedDisplay": 19, "intelligenceIndexIsEstimated": true, "slug": "kimi-k2-5-non-reasoning", "shortName": "Kimi K2.5 (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Kimi", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.131603336422614 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "longcat-2.0", "boardId": "aa_intelligence_index", "variantLabel": "LongCat 2.0", "score": 19.1123, "secondary": { "configurationRank": 234, "costPerTaskUsd": 0.059, "scoreRoundedDisplay": 19, "intelligenceIndexIsEstimated": false, "slug": "longcat-2-0", "shortName": "LongCat 2.0", "isReasoning": true, "deprecated": false, "modelCreatorName": "LongCat", "effort": null, "terminalBench40": 0, "scicode": 0.363425925925926, "hle": 0.336886005560704 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "deepseek-v4-flash", "boardId": "aa_intelligence_index", "variantLabel": "DeepSeek V4 Flash (non-reasoning)", "score": 18.8769, "secondary": { "configurationRank": 236, "costPerTaskUsd": null, "scoreRoundedDisplay": 19, "intelligenceIndexIsEstimated": true, "slug": "deepseek-v4-flash-0420-non-reasoning", "shortName": "DeepSeek V4 Flash (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.0778498609823911 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "mimo-v2.5-pro", "boardId": "aa_intelligence_index", "variantLabel": "MiMo-V2.5-Pro (non-reasoning)", "score": 18.29, "secondary": { "configurationRank": 243, "costPerTaskUsd": null, "scoreRoundedDisplay": 18, "intelligenceIndexIsEstimated": true, "slug": "mimo-v2-5-pro-non-reasoning", "shortName": "MiMo-V2.5-Pro (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "Xiaomi", "effort": "non-reasoning", "terminalBench40": null, "scicode": null, "hle": 0.147822057460612 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-6 Luna (non-reasoning)", "score": 18.2617, "secondary": { "configurationRank": 244, "costPerTaskUsd": 0.0115, "scoreRoundedDisplay": 18, "intelligenceIndexIsEstimated": false, "slug": "gpt-6-luna-non-reasoning", "shortName": "GPT-6 Luna (non-reasoning)", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenAI", "effort": "non-reasoning", "terminalBench40": 0.0151515151515152, "scicode": 0.430555555555556, "hle": 0.0857275254865616 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "step-3.5-flash", "boardId": "aa_intelligence_index", "variantLabel": "Step 3.5 Flash 2603", "score": 16.9641, "secondary": { "configurationRank": 256, "costPerTaskUsd": null, "scoreRoundedDisplay": 17, "intelligenceIndexIsEstimated": true, "slug": "step-3-5-flash", "shortName": "Step 3.5 Flash 2603", "isReasoning": true, "deprecated": true, "modelCreatorName": "StepFun", "effort": null, "terminalBench40": null, "scicode": null, "hle": 0.244670991658943 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "step-3.5-flash", "boardId": "aa_intelligence_index", "variantLabel": "Step 3.5 Flash", "score": 16.6169, "secondary": { "configurationRank": 263, "costPerTaskUsd": null, "scoreRoundedDisplay": 17, "intelligenceIndexIsEstimated": true, "slug": "step-3-5-flash-0202", "shortName": "Step 3.5 Flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "StepFun", "effort": null, "terminalBench40": null, "scicode": null, "hle": 0.211306765523633 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "Marked intelligenceIndexIsEstimated=true in AA embedded payload; retained as estimate, not treated as fully measured." }, { "model": "gpt-5.6-luna", "boardId": "aa_intelligence_index", "variantLabel": "GPT-5.6 Luna (non-reasoning)", "score": 15.527, "secondary": { "configurationRank": 275, "costPerTaskUsd": 0.0101, "scoreRoundedDisplay": 16, "intelligenceIndexIsEstimated": false, "slug": "gpt-5-6-luna-non-reasoning", "shortName": "GPT-5.6 Luna (non-reasoning)", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "effort": "non-reasoning", "terminalBench40": 0.0101010101010101, "scicode": 0.403935185185185, "hle": 0.072289156626506 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "mistral-medium-3.5", "boardId": "aa_intelligence_index", "variantLabel": "Mistral Medium 3.5", "score": 14.1889, "secondary": { "configurationRank": 296, "costPerTaskUsd": 0.502, "scoreRoundedDisplay": 14, "intelligenceIndexIsEstimated": false, "slug": "mistral-medium-3-5", "shortName": "Mistral Medium 3.5", "isReasoning": true, "deprecated": false, "modelCreatorName": "Mistral", "effort": null, "terminalBench40": 0, "scicode": 0.40162037037037, "hle": 0.137627432808156 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01" }, { "model": "claude-sonnet-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5.5 (max with fallback)", "score": 63.6364, "secondary": { "configurationRank": 1, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "claude-sonnet-5-5", "shortName": "Claude Sonnet 5.5 (max with fallback)", "terminalBench40Fraction": 0.636363636363636 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-astra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Astra (xhigh)", "score": 59.596, "secondary": { "configurationRank": 2, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-6-astra-xhigh", "shortName": "GPT-6 Astra (xhigh)", "terminalBench40Fraction": 0.595959595959596 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5.5 (max with fallback)", "score": 59.596, "secondary": { "configurationRank": 3, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "claude-opus-5-5", "shortName": "Claude Opus 5.5 (max with fallback)", "terminalBench40Fraction": 0.595959595959596 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5.5 (xhigh with fallback)", "score": 59.596, "secondary": { "configurationRank": 4, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "claude-opus-5-5-xhigh", "shortName": "Claude Opus 5.5 (xhigh with fallback)", "terminalBench40Fraction": 0.595959595959596 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-astra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Astra (max)", "score": 59.0909, "secondary": { "configurationRank": 5, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-6-astra", "shortName": "GPT-6 Astra (max)", "terminalBench40Fraction": 0.590909090909091 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5.5 (xhigh with fallback)", "score": 57.0707, "secondary": { "configurationRank": 6, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "claude-sonnet-5-5-xhigh", "shortName": "Claude Sonnet 5.5 (xhigh with fallback)", "terminalBench40Fraction": 0.570707070707071 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5.5 (high with fallback)", "score": 56.5657, "secondary": { "configurationRank": 8, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "claude-opus-5-5-high", "shortName": "Claude Opus 5.5 (high with fallback)", "terminalBench40Fraction": 0.565656565656566 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6.1-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6.1 Sol (max)", "score": 56.0606, "secondary": { "configurationRank": 9, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-6-1-sol", "shortName": "GPT-6.1 Sol (max)", "terminalBench40Fraction": 0.560606060606061 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-fable-5.1", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Fable 5.1 (xhigh with fallback)", "score": 55.0505, "secondary": { "configurationRank": 10, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "claude-fable-5-1-xhigh", "shortName": "Claude Fable 5.1 (xhigh with fallback)", "terminalBench40Fraction": 0.55050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6.1-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6.1 Sol (xhigh)", "score": 54.0404, "secondary": { "configurationRank": 11, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-6-1-sol-xhigh", "shortName": "GPT-6.1 Sol (xhigh)", "terminalBench40Fraction": 0.54040404040404 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-astra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Astra (high)", "score": 54.0404, "secondary": { "configurationRank": 12, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-6-astra-high", "shortName": "GPT-6 Astra (high)", "terminalBench40Fraction": 0.54040404040404 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5.5 (medium with fallback)", "score": 52.5253, "secondary": { "configurationRank": 13, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "claude-opus-5-5-medium", "shortName": "Claude Opus 5.5 (medium with fallback)", "terminalBench40Fraction": 0.525252525252525 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-fable-5.1", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Fable 5.1 (max with fallback)", "score": 52.0202, "secondary": { "configurationRank": 14, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "claude-fable-5-1", "shortName": "Claude Fable 5.1 (max with fallback)", "terminalBench40Fraction": 0.52020202020202 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-fable-5.1", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Fable 5.1 (high with fallback)", "score": 52.0202, "secondary": { "configurationRank": 15, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "claude-fable-5-1-high", "shortName": "Claude Fable 5.1 (high with fallback)", "terminalBench40Fraction": 0.52020202020202 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6.1-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6.1 Sol (high)", "score": 51.5152, "secondary": { "configurationRank": 16, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-6-1-sol-high", "shortName": "GPT-6.1 Sol (high)", "terminalBench40Fraction": 0.515151515151515 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-astra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Astra (medium)", "score": 49.4949, "secondary": { "configurationRank": 17, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-6-astra-medium", "shortName": "GPT-6 Astra (medium)", "terminalBench40Fraction": 0.494949494949495 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5 (max)", "score": 48.9899, "secondary": { "configurationRank": 18, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "claude-opus-5", "shortName": "Claude Opus 5 (max)", "terminalBench40Fraction": 0.48989898989899 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6.1-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6.1 Sol (medium)", "score": 47.9798, "secondary": { "configurationRank": 19, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-6-1-sol-medium", "shortName": "GPT-6.1 Sol (medium)", "terminalBench40Fraction": 0.47979797979798 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5 (xhigh)", "score": 46.4646, "secondary": { "configurationRank": 20, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "claude-opus-5-xhigh", "shortName": "Claude Opus 5 (xhigh)", "terminalBench40Fraction": 0.464646464646465 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5 (high)", "score": 45.9596, "secondary": { "configurationRank": 21, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "claude-opus-5-high", "shortName": "Claude Opus 5 (high)", "terminalBench40Fraction": 0.45959595959596 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-fable-5.1", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Fable 5.1 (medium with fallback)", "score": 44.9495, "secondary": { "configurationRank": 22, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "claude-fable-5-1-medium", "shortName": "Claude Fable 5.1 (medium with fallback)", "terminalBench40Fraction": 0.44949494949495 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5.5 (high with fallback)", "score": 43.9394, "secondary": { "configurationRank": 23, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "claude-sonnet-5-5-high", "shortName": "Claude Sonnet 5.5 (high with fallback)", "terminalBench40Fraction": 0.439393939393939 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Sol (max)", "score": 43.9394, "secondary": { "configurationRank": 24, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-6-sol", "shortName": "GPT-6 Sol (max)", "terminalBench40Fraction": 0.439393939393939 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-fable-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Fable 5 (with fallback)", "score": 42.4242, "secondary": { "configurationRank": 25, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "claude-fable-5", "shortName": "Claude Fable 5 (with fallback)", "terminalBench40Fraction": 0.424242424242424 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-astra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Astra (low)", "score": 41.9192, "secondary": { "configurationRank": 26, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-6-astra-low", "shortName": "GPT-6 Astra (low)", "terminalBench40Fraction": 0.419191919191919 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "glm-5.3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GLM-5.3 (max)", "score": 41.9192, "secondary": { "configurationRank": 27, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "glm-5-3", "shortName": "GLM-5.3 (max)", "terminalBench40Fraction": 0.419191919191919 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-fable-5.1", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Fable 5.1 (low with fallback)", "score": 40.404, "secondary": { "configurationRank": 28, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "claude-fable-5-1-low", "shortName": "Claude Fable 5.1 (low with fallback)", "terminalBench40Fraction": 0.404040404040404 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Sol (max)", "score": 39.899, "secondary": { "configurationRank": 29, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-5-6-sol", "shortName": "GPT-5.6 Sol (max)", "terminalBench40Fraction": 0.398989898989899 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.8-max-0902", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.8 Max (0902)", "score": 38.8889, "secondary": { "configurationRank": 30, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "qwen3-8-max", "shortName": "Qwen3.8 Max (0902)", "terminalBench40Fraction": 0.388888888888889 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-terra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Terra (max)", "score": 35.3535, "secondary": { "configurationRank": 31, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-5-6-terra", "shortName": "GPT-5.6 Terra (max)", "terminalBench40Fraction": 0.353535353535354 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "mimo-v2.6-pro", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - MiMo-V2.6-Pro", "score": 34.8485, "secondary": { "configurationRank": 32, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "mimo-v2-6-pro", "shortName": "MiMo-V2.6-Pro", "terminalBench40Fraction": 0.348484848484849 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "glm-5.3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GLM-5.3 (low)", "score": 34.8485, "secondary": { "configurationRank": 33, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "glm-5-3-low", "shortName": "GLM-5.3 (low)", "terminalBench40Fraction": 0.348484848484849 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5 (medium)", "score": 34.3434, "secondary": { "configurationRank": 34, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "claude-opus-5-medium", "shortName": "Claude Opus 5 (medium)", "terminalBench40Fraction": 0.343434343434343 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "step-5-preview", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Step 5 Preview", "score": 33.3333, "secondary": { "configurationRank": 35, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "step-5", "shortName": "Step 5 Preview", "terminalBench40Fraction": 0.333333333333333 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "muse-spark-1.3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Muse Spark 1.3 (max)", "score": 33.3333, "secondary": { "configurationRank": 36, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "muse-spark-1-3", "shortName": "Muse Spark 1.3 (max)", "terminalBench40Fraction": 0.333333333333333 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "glm-5.3-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GLM-5.3-Flash", "score": 32.8283, "secondary": { "configurationRank": 37, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "glm-5-3-flash", "shortName": "GLM-5.3-Flash", "terminalBench40Fraction": 0.328282828282828 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5.5 (low with fallback)", "score": 31.3131, "secondary": { "configurationRank": 38, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "claude-opus-5-5-low", "shortName": "Claude Opus 5.5 (low with fallback)", "terminalBench40Fraction": 0.313131313131313 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6.1-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6.1 Sol (low)", "score": 30.8081, "secondary": { "configurationRank": 39, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-6-1-sol-low", "shortName": "GPT-6.1 Sol (low)", "terminalBench40Fraction": 0.308080808080808 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Sol (xhigh)", "score": 30.303, "secondary": { "configurationRank": 40, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-6-sol-xhigh", "shortName": "GPT-6 Sol (xhigh)", "terminalBench40Fraction": 0.303030303030303 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5.5 (medium with fallback)", "score": 29.798, "secondary": { "configurationRank": 41, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "claude-sonnet-5-5-medium", "shortName": "Claude Sonnet 5.5 (medium with fallback)", "terminalBench40Fraction": 0.297979797979798 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4.1-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4.1 Flash (max)", "score": 26.7677, "secondary": { "configurationRank": 42, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "deepseek-v4-1-flash", "shortName": "DeepSeek V4.1 Flash (max)", "terminalBench40Fraction": 0.267676767676768 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 5 (low)", "score": 26.2626, "secondary": { "configurationRank": 43, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "claude-opus-5-low", "shortName": "Claude Opus 5 (low)", "terminalBench40Fraction": 0.262626262626263 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Sol (high)", "score": 26.2626, "secondary": { "configurationRank": 44, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-6-sol-high", "shortName": "GPT-6 Sol (high)", "terminalBench40Fraction": 0.262626262626263 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.7", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.7 (xhigh)", "score": 25.7576, "secondary": { "configurationRank": 45, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "grok-4-7", "shortName": "Grok 4.7 (xhigh)", "terminalBench40Fraction": 0.257575757575758 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.7", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.7 (high)", "score": 24.7475, "secondary": { "configurationRank": 47, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "grok-4-7-high", "shortName": "Grok 4.7 (high)", "terminalBench40Fraction": 0.247474747474747 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Sol (xhigh)", "score": 24.7475, "secondary": { "configurationRank": 48, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-5-6-sol-xhigh", "shortName": "GPT-5.6 Sol (xhigh)", "terminalBench40Fraction": 0.247474747474747 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "mimo-v2.6-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - MiMo-V2.6-Flash", "score": 22.7273, "secondary": { "configurationRank": 49, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "mimo-v2-6-flash", "shortName": "MiMo-V2.6-Flash", "terminalBench40Fraction": 0.227272727272727 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-opus-4.8", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Opus 4.8 (max)", "score": 21.7172, "secondary": { "configurationRank": 50, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "claude-opus-4-8", "shortName": "Claude Opus 4.8 (max)", "terminalBench40Fraction": 0.217171717171717 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.6", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.6 (high)", "score": 21.2121, "secondary": { "configurationRank": 51, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "grok-4-6", "shortName": "Grok 4.6 (high)", "terminalBench40Fraction": 0.212121212121212 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Sol (high)", "score": 20.7071, "secondary": { "configurationRank": 52, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-5-6-sol-high", "shortName": "GPT-5.6 Sol (high)", "terminalBench40Fraction": 0.207070707070707 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5.5 (low with fallback)", "score": 20.7071, "secondary": { "configurationRank": 53, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "claude-sonnet-5-5-low", "shortName": "Claude Sonnet 5.5 (low with fallback)", "terminalBench40Fraction": 0.207070707070707 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gemini-3.8-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Gemini 3.8 Flash (medium)", "score": 19.697, "secondary": { "configurationRank": 54, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gemini-3-8-flash-medium", "shortName": "Gemini 3.8 Flash (medium)", "terminalBench40Fraction": 0.196969696969697 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gemini-3.8-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Gemini 3.8 Flash (high)", "score": 19.697, "secondary": { "configurationRank": 55, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gemini-3-8-flash", "shortName": "Gemini 3.8 Flash (high)", "terminalBench40Fraction": 0.196969696969697 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.8-max", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.8 Max", "score": 18.6869, "secondary": { "configurationRank": 56, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "qwen3-8-max-0803", "shortName": "Qwen3.8 Max", "terminalBench40Fraction": 0.186868686868687 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Sol (medium)", "score": 18.6869, "secondary": { "configurationRank": 57, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-6-sol-medium", "shortName": "GPT-6 Sol (medium)", "terminalBench40Fraction": 0.186868686868687 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.6", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.6 (xhigh)", "score": 17.1717, "secondary": { "configurationRank": 58, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "grok-4-6-xhigh", "shortName": "Grok 4.6 (xhigh)", "terminalBench40Fraction": 0.171717171717172 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "muse-spark-1.3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Muse Spark 1.3 (xhigh)", "score": 16.6667, "secondary": { "configurationRank": 59, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "muse-spark-1-3-xhigh", "shortName": "Muse Spark 1.3 (xhigh)", "terminalBench40Fraction": 0.166666666666667 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.5 (xhigh)", "score": 14.6465, "secondary": { "configurationRank": 60, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-5-5", "shortName": "GPT-5.5 (xhigh)", "terminalBench40Fraction": 0.146464646464646 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Sol (medium)", "score": 14.6465, "secondary": { "configurationRank": 61, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-5-6-sol-medium", "shortName": "GPT-5.6 Sol (medium)", "terminalBench40Fraction": 0.146464646464646 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-pro", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Pro (max)", "score": 14.6465, "secondary": { "configurationRank": 62, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "deepseek-v4-pro-0424", "shortName": "DeepSeek V4 Pro (max)", "terminalBench40Fraction": 0.146464646464646 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5 (max)", "score": 14.1414, "secondary": { "configurationRank": 63, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "claude-sonnet-5", "shortName": "Claude Sonnet 5 (max)", "terminalBench40Fraction": 0.141414141414141 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-pro", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Pro 0813 (max)", "score": 14.1414, "secondary": { "configurationRank": 64, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "deepseek-v4-pro", "shortName": "DeepSeek V4 Pro 0813 (max)", "terminalBench40Fraction": 0.141414141414141 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gemini-3.7-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Gemini 3.7 Flash (high)", "score": 13.6364, "secondary": { "configurationRank": 65, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gemini-3-7-flash", "shortName": "Gemini 3.7 Flash (high)", "terminalBench40Fraction": 0.136363636363636 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.6", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.6 (medium)", "score": 13.1313, "secondary": { "configurationRank": 66, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "grok-4-6-medium", "shortName": "Grok 4.6 (medium)", "terminalBench40Fraction": 0.131313131313131 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Sol (non-reasoning)", "score": 13.1313, "secondary": { "configurationRank": 67, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "gpt-6-sol-non-reasoning", "shortName": "GPT-6 Sol (non-reasoning)", "terminalBench40Fraction": 0.131313131313131 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "kimi-k3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Kimi K3 (low)", "score": 12.6263, "secondary": { "configurationRank": 68, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "kimi-k3-low", "shortName": "Kimi K3 (low)", "terminalBench40Fraction": 0.126262626262626 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Luna (max)", "score": 12.6263, "secondary": { "configurationRank": 69, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-6-luna", "shortName": "GPT-6 Luna (max)", "terminalBench40Fraction": 0.126262626262626 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "kimi-k3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Kimi K3 (max)", "score": 12.6263, "secondary": { "configurationRank": 71, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "kimi-k3", "shortName": "Kimi K3 (max)", "terminalBench40Fraction": 0.126262626262626 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-flash-vision-exp", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Flash Vision (max)", "score": 12.1212, "secondary": { "configurationRank": 72, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "deepseek-v4-flash-vision", "shortName": "DeepSeek V4 Flash Vision (max)", "terminalBench40Fraction": 0.121212121212121 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Flash 0731 (max)", "score": 12.1212, "secondary": { "configurationRank": 73, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "deepseek-v4-flash", "shortName": "DeepSeek V4 Flash 0731 (max)", "terminalBench40Fraction": 0.121212121212121 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Luna (max)", "score": 11.6162, "secondary": { "configurationRank": 74, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "gpt-5-6-luna", "shortName": "GPT-5.6 Luna (max)", "terminalBench40Fraction": 0.116161616161616 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.5 (high)", "score": 10.6061, "secondary": { "configurationRank": 76, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "grok-4-5", "shortName": "Grok 4.5 (high)", "terminalBench40Fraction": 0.106060606060606 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-terra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Terra (xhigh)", "score": 10.101, "secondary": { "configurationRank": 77, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-5-6-terra-xhigh", "shortName": "GPT-5.6 Terra (xhigh)", "terminalBench40Fraction": 0.101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gemini-3.8-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Gemini 3.8 Flash (low)", "score": 10.101, "secondary": { "configurationRank": 78, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gemini-3-8-flash-low", "shortName": "Gemini 3.8 Flash (low)", "terminalBench40Fraction": 0.101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Sol (low)", "score": 9.0909, "secondary": { "configurationRank": 79, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-6-sol-low", "shortName": "GPT-6 Sol (low)", "terminalBench40Fraction": 0.0909090909090909 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.5 (high)", "score": 9.0909, "secondary": { "configurationRank": 80, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-5-5-high", "shortName": "GPT-5.5 (high)", "terminalBench40Fraction": 0.0909090909090909 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Luna (xhigh)", "score": 8.0808, "secondary": { "configurationRank": 81, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-6-luna-xhigh", "shortName": "GPT-6 Luna (xhigh)", "terminalBench40Fraction": 0.0808080808080808 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "muse-spark-1.2", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Muse Spark 1.2 (xhigh)", "score": 7.0707, "secondary": { "configurationRank": 82, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "muse-spark-1-2", "shortName": "Muse Spark 1.2 (xhigh)", "terminalBench40Fraction": 0.0707070707070707 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gemini-3.6-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Gemini 3.6 Flash (high)", "score": 7.0707, "secondary": { "configurationRank": 83, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gemini-3-6-flash", "shortName": "Gemini 3.6 Flash (high)", "terminalBench40Fraction": 0.0707070707070707 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5 (xhigh)", "score": 7.0707, "secondary": { "configurationRank": 84, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "claude-sonnet-5-xhigh", "shortName": "Claude Sonnet 5 (xhigh)", "terminalBench40Fraction": 0.0707070707070707 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4.1-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4.1 Flash (non-reasoning)", "score": 5.5556, "secondary": { "configurationRank": 87, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "deepseek-v4-1-flash-non-reasoning", "shortName": "DeepSeek V4.1 Flash (non-reasoning)", "terminalBench40Fraction": 0.0555555555555556 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.8-27b", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.8 27B (xhigh)", "score": 5.5556, "secondary": { "configurationRank": 88, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "qwen3-8-27b", "shortName": "Qwen3.8 27B (xhigh)", "terminalBench40Fraction": 0.0555555555555556 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.8-27b", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.8 27B (medium)", "score": 5.0505, "secondary": { "configurationRank": 89, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "qwen3-8-27b-medium", "shortName": "Qwen3.8 27B (medium)", "terminalBench40Fraction": 0.0505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.5 (medium)", "score": 5.0505, "secondary": { "configurationRank": 90, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-5-5-medium", "shortName": "GPT-5.5 (medium)", "terminalBench40Fraction": 0.0505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5 (high)", "score": 5.0505, "secondary": { "configurationRank": 91, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "claude-sonnet-5-high", "shortName": "Claude Sonnet 5 (high)", "terminalBench40Fraction": 0.0505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Luna (high)", "score": 4.5455, "secondary": { "configurationRank": 92, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-6-luna-high", "shortName": "GPT-6 Luna (high)", "terminalBench40Fraction": 0.0454545454545455 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gemini-3.1-pro", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Gemini 3.1 Pro Preview", "score": 4.0404, "secondary": { "configurationRank": 93, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "gemini-3-1-pro-preview", "shortName": "Gemini 3.1 Pro Preview", "terminalBench40Fraction": 0.0404040404040404 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Luna (xhigh)", "score": 3.5354, "secondary": { "configurationRank": 94, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "gpt-5-6-luna-xhigh", "shortName": "GPT-5.6 Luna (xhigh)", "terminalBench40Fraction": 0.0353535353535354 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Flash (high)", "score": 3.0303, "secondary": { "configurationRank": 96, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "deepseek-v4-flash-0420-high", "shortName": "DeepSeek V4 Flash (high)", "terminalBench40Fraction": 0.0303030303030303 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "grok-4.6", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Grok 4.6 (low)", "score": 3.0303, "secondary": { "configurationRank": 97, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "grok-4-6-low", "shortName": "Grok 4.6 (low)", "terminalBench40Fraction": 0.0303030303030303 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5 (low)", "score": 2.5253, "secondary": { "configurationRank": 99, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "claude-sonnet-5-low", "shortName": "Claude Sonnet 5 (low)", "terminalBench40Fraction": 0.0252525252525253 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Luna (medium)", "score": 2.5253, "secondary": { "configurationRank": 100, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-6-luna-medium", "shortName": "GPT-6 Luna (medium)", "terminalBench40Fraction": 0.0252525252525253 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.8-27b", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.8 27B (low)", "score": 2.5253, "secondary": { "configurationRank": 101, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "qwen3-8-27b-low", "shortName": "Qwen3.8 27B (low)", "terminalBench40Fraction": 0.0252525252525253 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Luna (high)", "score": 2.5253, "secondary": { "configurationRank": 102, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-5-6-luna-high", "shortName": "GPT-5.6 Luna (high)", "terminalBench40Fraction": 0.0252525252525253 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-flash", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Flash (max)", "score": 2.5253, "secondary": { "configurationRank": 103, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "deepseek-v4-flash-0420", "shortName": "DeepSeek V4 Flash (max)", "terminalBench40Fraction": 0.0252525252525253 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "minimax-m3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - MiniMax-M3", "score": 2.0202, "secondary": { "configurationRank": 104, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "minimax-m3", "shortName": "MiniMax-M3", "terminalBench40Fraction": 0.0202020202020202 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "glm-5.1", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GLM-5.1", "score": 2.0202, "secondary": { "configurationRank": 105, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "glm-5-1", "shortName": "GLM-5.1", "terminalBench40Fraction": 0.0202020202020202 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Claude Sonnet 5 (medium)", "score": 2.0202, "secondary": { "configurationRank": 107, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "claude-sonnet-5-medium", "shortName": "Claude Sonnet 5 (medium)", "terminalBench40Fraction": 0.0202020202020202 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Luna (non-reasoning)", "score": 1.5152, "secondary": { "configurationRank": 108, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "gpt-6-luna-non-reasoning", "shortName": "GPT-6 Luna (non-reasoning)", "terminalBench40Fraction": 0.0151515151515152 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.7-max", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.7 Max", "score": 1.5152, "secondary": { "configurationRank": 109, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "qwen3-7-max", "shortName": "Qwen3.7 Max", "terminalBench40Fraction": 0.0151515151515152 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-terra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Terra (high)", "score": 1.5152, "secondary": { "configurationRank": 110, "agentHarness": "Artificial Analysis", "reasoningEffort": "high", "slug": "gpt-5-6-terra-high", "shortName": "GPT-5.6 Terra (high)", "terminalBench40Fraction": 0.0151515151515152 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-terra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Terra (low)", "score": 1.5152, "secondary": { "configurationRank": 112, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-5-6-terra-low", "shortName": "GPT-5.6 Terra (low)", "terminalBench40Fraction": 0.0151515151515152 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "deepseek-v4-pro", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - DeepSeek V4 Pro 0813 (non-reasoning)", "score": 1.5152, "secondary": { "configurationRank": 113, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "deepseek-v4-pro-non-reasoning", "shortName": "DeepSeek V4 Pro 0813 (non-reasoning)", "terminalBench40Fraction": 0.0151515151515152 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-sol", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Sol (low)", "score": 1.0101, "secondary": { "configurationRank": 114, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-5-6-sol-low", "shortName": "GPT-5.6 Sol (low)", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "inkling", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Inkling (xhigh)", "score": 1.0101, "secondary": { "configurationRank": 116, "agentHarness": "Artificial Analysis", "reasoningEffort": "xhigh", "slug": "inkling", "shortName": "Inkling (xhigh)", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-terra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Terra (medium)", "score": 1.0101, "secondary": { "configurationRank": 118, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-5-6-terra-medium", "shortName": "GPT-5.6 Terra (medium)", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "inkling-small", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Inkling Small", "score": 1.0101, "secondary": { "configurationRank": 120, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "inkling-small", "shortName": "Inkling Small", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "kimi-k2.7-code", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Kimi K2.7 Code", "score": 1.0101, "secondary": { "configurationRank": 121, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "kimi-k2-7-code", "shortName": "Kimi K2.7 Code", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.7-plus", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.7 Plus", "score": 1.0101, "secondary": { "configurationRank": 122, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "qwen3-7-plus", "shortName": "Qwen3.7 Plus", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Luna (non-reasoning)", "score": 1.0101, "secondary": { "configurationRank": 123, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "gpt-5-6-luna-non-reasoning", "shortName": "GPT-5.6 Luna (non-reasoning)", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "glm-5.2", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GLM-5.2 (max)", "score": 1.0101, "secondary": { "configurationRank": 124, "agentHarness": "Artificial Analysis", "reasoningEffort": "max", "slug": "glm-5-2", "shortName": "GLM-5.2 (max)", "terminalBench40Fraction": 0.0101010101010101 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "kimi-k2.6", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Kimi K2.6", "score": 0.5051, "secondary": { "configurationRank": 126, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "kimi-k2-6", "shortName": "Kimi K2.6", "terminalBench40Fraction": 0.00505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Luna (medium)", "score": 0.5051, "secondary": { "configurationRank": 133, "agentHarness": "Artificial Analysis", "reasoningEffort": "medium", "slug": "gpt-5-6-luna-medium", "shortName": "GPT-5.6 Luna (medium)", "terminalBench40Fraction": 0.00505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "nemotron-3-ultra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Nemotron 3 Ultra", "score": 0.5051, "secondary": { "configurationRank": 135, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "nvidia-nemotron-3-ultra-550b-a55b", "shortName": "Nemotron 3 Ultra", "terminalBench40Fraction": 0.00505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-terra", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Terra (non-reasoning)", "score": 0.5051, "secondary": { "configurationRank": 136, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "gpt-5-6-terra-non-reasoning", "shortName": "GPT-5.6 Terra (non-reasoning)", "terminalBench40Fraction": 0.00505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "hy3", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Hy3", "score": 0.5051, "secondary": { "configurationRank": 137, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "hy3", "shortName": "Hy3", "terminalBench40Fraction": 0.00505050505050505 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "mimo-v2.5-pro", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - MiMo-V2.5-Pro", "score": 0.0, "secondary": { "configurationRank": 141, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "mimo-v2-5-pro", "shortName": "MiMo-V2.5-Pro", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-5.6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-5.6 Luna (low)", "score": 0.0, "secondary": { "configurationRank": 143, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-5-6-luna-low", "shortName": "GPT-5.6 Luna (low)", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "mimo-v2.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - MiMo-V2.5", "score": 0.0, "secondary": { "configurationRank": 153, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "mimo-v2-5-0424", "shortName": "MiMo-V2.5", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "qwen3.8-27b", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Qwen3.8 27B (non-reasoning)", "score": 0.0, "secondary": { "configurationRank": 161, "agentHarness": "Artificial Analysis", "reasoningEffort": "non-reasoning", "slug": "qwen3-8-27b-non-reasoning", "shortName": "Qwen3.8 27B (non-reasoning)", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "minimax-m2.7", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - MiniMax-M2.7", "score": 0.0, "secondary": { "configurationRank": 169, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "minimax-m2-7", "shortName": "MiniMax-M2.7", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "longcat-2.0", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - LongCat 2.0", "score": 0.0, "secondary": { "configurationRank": 171, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "longcat-2-0", "shortName": "LongCat 2.0", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "gpt-6-luna", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - GPT-6 Luna (low)", "score": 0.0, "secondary": { "configurationRank": 198, "agentHarness": "Artificial Analysis", "reasoningEffort": "low", "slug": "gpt-6-luna-low", "shortName": "GPT-6 Luna (low)", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "mistral-medium-3.5", "boardId": "aa_terminal_bench_4", "variantLabel": "Artificial Analysis - Mistral Medium 3.5", "score": 0.0, "secondary": { "configurationRank": 202, "agentHarness": "Artificial Analysis", "reasoningEffort": null, "slug": "mistral-medium-3-5", "shortName": "Mistral Medium 3.5", "terminalBench40Fraction": 0 }, "source": "https://artificialanalysis.ai/leaderboards/models", "checkedAt": "2026-10-01", "note": "AA 自家 harness 独立实测的 Terminal-Bench 4.0（models 榜载荷 terminalBench40 字段，fraction×100）；与 tbench.ai 官方榜分开计分。" }, { "model": "claude-sonnet-5.5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Sonnet 5.5 (max)", "score": 68.3578, "secondary": { "configurationRank": 1, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-max", "displayModel": "Sonnet 5.5 (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 71.9764, "terminalBenchV4": 66.1616, "sweAtlasQnA": 66.9355, "meanCostUsdPerTask": 14.1908, "agentWallTimeSec": 5244.9654, "meanSteps": 265.9307, "meanTotalTokens": 27742337.2338, "cacheHitRate": 0.9506, "hostConfigId": "a2c87c062f3cef73d8525e7578f14742", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-opus-5.5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Opus 5.5 (max)", "score": 65.9886, "secondary": { "configurationRank": 2, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-opus-5-5", "displayModel": "Opus 5.5 (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 68.4366, "terminalBenchV4": 63.1313, "sweAtlasQnA": 66.3978, "meanCostUsdPerTask": 13.0364, "agentWallTimeSec": 3867.0923, "meanSteps": 155.0957, "meanTotalTokens": 15551383.22, "cacheHitRate": 0.95, "hostConfigId": "ab5074e404be48bbd4ac3c488ad9a3e2", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6.1-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6.1 Sol (xhigh) ({'reasoning_effort': 'xhigh'})", "score": 62.9078, "secondary": { "configurationRank": 4, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "displayModel": "GPT-6.1 Sol (xhigh)", "isUnavailable": false, "variantOf": null, "deepSWE": 73.1563, "terminalBenchV4": 54.5455, "sweAtlasQnA": 61.0215, "meanCostUsdPerTask": 1.0397, "agentWallTimeSec": 931.3823, "meanSteps": 40.055, "meanTotalTokens": 3212528.1095, "cacheHitRate": 0.9328, "hostConfigId": "ed8c1d0b0a2fe4560bbfa0a9f47e8941", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-sonnet-5.5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Sonnet 5.5 (xhigh)", "score": 62.8714, "secondary": { "configurationRank": 5, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-xhigh", "displayModel": "Sonnet 5.5 (xhigh)", "isUnavailable": false, "variantOf": null, "deepSWE": 68.4366, "terminalBenchV4": 58.0808, "sweAtlasQnA": 62.0968, "meanCostUsdPerTask": 3.3334, "agentWallTimeSec": 1619.0135, "meanSteps": 78.3168, "meanTotalTokens": 7087678.396, "cacheHitRate": 0.9575, "hostConfigId": "666605c66ad569c503ce53c01682e285", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-fable-5.1", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Fable 5.1 (max) (with fallback)", "score": 62.2225, "secondary": { "configurationRank": 6, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-fable-5-1", "displayModel": "Fable 5.1 (max) (with fallback)", "isUnavailable": false, "variantOf": null, "deepSWE": 64.3068, "terminalBenchV4": 57.5758, "sweAtlasQnA": 64.7849, "meanCostUsdPerTask": 12.3881, "agentWallTimeSec": 2089.9001, "meanSteps": 36.6568, "meanTotalTokens": 5715253.8592, "cacheHitRate": 0.916, "hostConfigId": "29a1cdad0d140781d05b9b4746cfdd0d", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } }, "swe-atlas-qna": { "min": { "version": "2.1.259", "dateReleased": "2026-09-02" }, "max": { "version": "2.1.259", "dateReleased": "2026-09-02" } }, "terminal-bench-v4": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-fable-5.1", "boardId": "aa_coding_agent_index", "variantLabel": "Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium", "score": 61.6826, "secondary": { "configurationRank": 7, "agentHarness": "Devin Fusion CLI", "provider": "cognition", "hostModelSlug": "cognition_fusion-claude-fable-5-1-xhigh-sidekick-penguin-medium", "displayModel": "Claude Fable 5.1 XHigh + SWE-2 Medium", "isUnavailable": false, "variantOf": null, "deepSWE": 63.1268, "terminalBenchV4": 56.0606, "sweAtlasQnA": 65.8602, "meanCostUsdPerTask": 7.8971, "agentWallTimeSec": 2146.9921, "meanSteps": 99.5314, "meanTotalTokens": 9691548.1738, "cacheHitRate": 0.9585, "hostConfigId": "34970c963bdfd318e46205514d46ea4f", "versions": { "deep-swe-v1.1": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6-astra", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6 Astra (max)", "score": 61.645, "secondary": { "configurationRank": 8, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_vega-alpha", "displayModel": "GPT-6 Astra (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 67.5516, "terminalBenchV4": 55.5556, "sweAtlasQnA": 61.828, "meanCostUsdPerTask": 7.4671, "agentWallTimeSec": 1762.033, "meanSteps": 39.3784, "meanTotalTokens": 3346611.0122, "cacheHitRate": 0.9336, "hostConfigId": "9e810ff3c6cce5ebaaff7c744c687a47", "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.151.0", "dateReleased": "2026-08-29" }, "max": { "version": "0.151.0", "dateReleased": "2026-08-29" } }, "terminal-bench-v4": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6.1-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6.1 Sol (medium) ({'reasoning_effort': 'medium'})", "score": 61.4147, "secondary": { "configurationRank": 9, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "displayModel": "GPT-6.1 Sol (medium)", "isUnavailable": false, "variantOf": null, "deepSWE": 71.9764, "terminalBenchV4": 51.5152, "sweAtlasQnA": 60.7527, "meanCostUsdPerTask": 0.7047, "agentWallTimeSec": 651.3229, "meanSteps": 37.989, "meanTotalTokens": 2298278.7454, "cacheHitRate": 0.924, "hostConfigId": "93acd59f31f8e5478686d650aa650f79", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6.1-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6.1 Sol (high) ({'reasoning_effort': 'high'})", "score": 60.1492, "secondary": { "configurationRank": 10, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "displayModel": "GPT-6.1 Sol (high)", "isUnavailable": false, "variantOf": null, "deepSWE": 70.5015, "terminalBenchV4": 50.0, "sweAtlasQnA": 59.9462, "meanCostUsdPerTask": 0.8892, "agentWallTimeSec": 799.7652, "meanSteps": 38.9791, "meanTotalTokens": 2819911.9904, "cacheHitRate": 0.9278, "hostConfigId": "4aee8aa2d4a0bad6068644f40edae235", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6.1-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6.1 Sol (max)", "score": 60.1475, "secondary": { "configurationRank": 11, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "displayModel": "GPT-6.1 Sol (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 69.6165, "terminalBenchV4": 53.0303, "sweAtlasQnA": 57.7957, "meanCostUsdPerTask": 1.5541, "agentWallTimeSec": 1463.0464, "meanSteps": 40.4565, "meanTotalTokens": 3999501.6108, "cacheHitRate": 0.9282, "hostConfigId": "ea296ce785422d3b97cfff4427f2e0d2", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-opus-5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Opus 5 (max)", "score": 59.7264, "secondary": { "configurationRank": 12, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-opus-5", "displayModel": "Opus 5 (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 62.5369, "terminalBenchV4": 54.5455, "sweAtlasQnA": 62.0968, "meanCostUsdPerTask": 10.7892, "agentWallTimeSec": 2515.5402, "meanSteps": 152.1304, "meanTotalTokens": 11374850.489, "cacheHitRate": 0.9698, "hostConfigId": "409518818f1e9ad21995d70631ab05b6", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } }, "swe-atlas-qna": { "min": { "version": "2.1.218", "dateReleased": "2026-07-22" }, "max": { "version": "2.1.227", "dateReleased": "2026-08-10" } }, "terminal-bench-v4": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6-astra", "boardId": "aa_coding_agent_index", "variantLabel": "Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium", "score": 58.8884, "secondary": { "configurationRank": 13, "agentHarness": "Devin Fusion CLI", "provider": "cognition", "hostModelSlug": "cognition_fusion-gpt-6-astra-xhigh-sidekick-penguin-medium", "displayModel": "GPT-6 Astra XHigh + SWE-2 Medium", "isUnavailable": false, "variantOf": null, "deepSWE": 67.2566, "terminalBenchV4": 50.0, "sweAtlasQnA": 59.4086, "meanCostUsdPerTask": 4.5365, "agentWallTimeSec": 1481.896, "meanSteps": 78.5512, "meanTotalTokens": 6074134.495, "cacheHitRate": 0.9201, "hostConfigId": "fa2595093fa640e73d072e7d1aa7df82", "versions": { "deep-swe-v1.1": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6.1-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6.1 Sol (low) ({'reasoning_effort': 'low'})", "score": 57.2163, "secondary": { "configurationRank": 14, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "displayModel": "GPT-6.1 Sol (low)", "isUnavailable": false, "variantOf": null, "deepSWE": 67.5516, "terminalBenchV4": 48.9899, "sweAtlasQnA": 55.1075, "meanCostUsdPerTask": 0.4988, "agentWallTimeSec": 517.8139, "meanSteps": 35.7404, "meanTotalTokens": 1713818.9182, "cacheHitRate": 0.9136, "hostConfigId": "fa589dad83ffcdd6cce72a467e552063", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6 Sol (max)", "score": 56.6626, "secondary": { "configurationRank": 15, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-6-sol", "displayModel": "GPT-6 Sol (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 69.0265, "terminalBenchV4": 43.4343, "sweAtlasQnA": 57.5269, "meanCostUsdPerTask": 2.9897, "agentWallTimeSec": 1338.0527, "meanSteps": 86.5501, "meanTotalTokens": 9840777.7635, "cacheHitRate": 0.964, "hostConfigId": "7a0dfd2a7a9d93056f860d4675586d66", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "grok-4.7", "boardId": "aa_coding_agent_index", "variantLabel": "Grok Build - Grok 4.7 (xhigh)", "score": 56.2676, "secondary": { "configurationRank": 16, "agentHarness": "Grok Build", "provider": "xai", "hostModelSlug": "xai_grok-bop-o86-xhigh", "displayModel": "Grok 4.7 (xhigh)", "isUnavailable": false, "variantOf": null, "deepSWE": 72.5664, "terminalBenchV4": 33.3333, "sweAtlasQnA": 62.9032, "meanCostUsdPerTask": 8.8175, "agentWallTimeSec": 2354.3677, "meanSteps": 162.6458, "meanTotalTokens": 14324724.2838, "cacheHitRate": 0.9389, "hostConfigId": "f927d9c41b5c3021996eb14c3dd1341e", "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-sonnet-5.5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Sonnet 5.5 (high)", "score": 55.0125, "secondary": { "configurationRank": 17, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-high", "displayModel": "Sonnet 5.5 (high)", "isUnavailable": false, "variantOf": null, "deepSWE": 66.6667, "terminalBenchV4": 41.9192, "sweAtlasQnA": 56.4516, "meanCostUsdPerTask": 1.2355, "agentWallTimeSec": 735.181, "meanSteps": 36.7723, "meanTotalTokens": 2652380.5655, "cacheHitRate": 0.9478, "hostConfigId": "33e80f383dc644527161435b076798ad", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-5.6-sol", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-5.6 Sol (max) ({'reasoning_effort': 'max'})", "score": 54.5591, "secondary": { "configurationRank": 18, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-5-6-sol", "displayModel": "GPT-5.6 Sol (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 72.2714, "terminalBenchV4": 37.3737, "sweAtlasQnA": 54.0323, "meanCostUsdPerTask": 6.3533, "agentWallTimeSec": 1237.6964, "meanSteps": 116.3949, "meanTotalTokens": 10170136.6326, "cacheHitRate": 0.9458, "hostConfigId": "709f801275dedae5675b1f56e97c9df3", "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.139.0", "dateReleased": "2026-06-09" }, "max": { "version": "0.147.0", "dateReleased": "2026-08-07" } }, "terminal-bench-v4": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "muse-spark-1.3", "boardId": "aa_coding_agent_index", "variantLabel": "Muse Code - Muse Spark 1.3 (max)", "score": 54.3027, "secondary": { "configurationRank": 19, "agentHarness": "Muse Code", "provider": "meta", "hostModelSlug": "meta_aa_glacier135", "displayModel": "Muse Spark 1.3 (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 71.6814, "terminalBenchV4": 31.8182, "sweAtlasQnA": 59.4086, "meanCostUsdPerTask": 3.9775, "agentWallTimeSec": 1106.8534, "meanSteps": 185.2382, "meanTotalTokens": 16709444.5737, "cacheHitRate": 0.9713, "hostConfigId": "f2de07373c3626f1b9269a6ecad7804c", "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "glm-5.3", "boardId": "aa_coding_agent_index", "variantLabel": "Opencode - GLM-5.3 ({'reasoning_effort': 'max'})", "score": 53.5548, "secondary": { "configurationRank": 20, "agentHarness": "Opencode", "provider": "zai", "hostModelSlug": "zai_glm-5-3", "displayModel": "GLM-5.3", "isUnavailable": false, "variantOf": null, "deepSWE": 61.3569, "terminalBenchV4": 39.899, "sweAtlasQnA": 59.4086, "meanCostUsdPerTask": 4.2414, "agentWallTimeSec": 2887.9557, "meanSteps": 107.0495, "meanTotalTokens": 14276424.4268, "cacheHitRate": 0.9746, "hostConfigId": "45493c2782f1bd94f6d30d4911f317a0", "versions": { "deep-swe-v1.1": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "kimi-k3", "boardId": "aa_coding_agent_index", "variantLabel": "Kimi Code CLI - Kimi K3", "score": 51.9259, "secondary": { "configurationRank": 21, "agentHarness": "Kimi Code CLI", "provider": "moonshotai", "hostModelSlug": "moonshot_kimi-k3", "displayModel": "Kimi K3", "isUnavailable": false, "variantOf": null, "deepSWE": 68.4366, "terminalBenchV4": 21.2121, "sweAtlasQnA": 66.129, "meanCostUsdPerTask": 5.0472, "agentWallTimeSec": 3689.3442, "meanSteps": 173.3603, "meanTotalTokens": 8481121.5336, "cacheHitRate": 0.95, "hostConfigId": "55b029d6971b4d378335bb168eb98b3b", "versions": { "deep-swe-v1.1": { "min": { "version": "0.36.0", "dateReleased": "2026-08-13" }, "max": { "version": "0.36.1", "dateReleased": "2026-08-14" } }, "swe-atlas-qna": { "min": { "version": "0.26.0", "dateReleased": "2026-07-16" }, "max": { "version": "0.26.0", "dateReleased": "2026-07-16" } }, "terminal-bench-v4": { "min": { "version": "0.36.1", "dateReleased": "2026-08-14" }, "max": { "version": "0.36.1", "dateReleased": "2026-08-14" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "muse-spark-1.3", "boardId": "aa_coding_agent_index", "variantLabel": "Muse Code - Muse Spark 1.3 (xhigh)", "score": 48.2993, "secondary": { "configurationRank": 22, "agentHarness": "Muse Code 1.0.2 RC", "provider": "meta", "hostModelSlug": "meta_joyful-jello138", "displayModel": "Muse Spark 1.3 (xhigh)", "isUnavailable": false, "variantOf": null, "deepSWE": 73.1563, "terminalBenchV4": 17.1717, "sweAtlasQnA": 54.5699, "meanCostUsdPerTask": 3.4678, "agentWallTimeSec": 1098.2029, "meanSteps": 187.4895, "meanTotalTokens": 16237094.352, "cacheHitRate": 0.9662, "hostConfigId": "cd60470c35fd626a116af80fd5915dfa", "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } }, "swe-atlas-qna": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } }, "terminal-bench-v4": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "grok-4.6", "boardId": "aa_coding_agent_index", "variantLabel": "Grok Build - Grok 4.6 (xhigh)", "score": 46.969, "secondary": { "configurationRank": 23, "agentHarness": "Grok Build", "provider": "xai", "hostModelSlug": "xai_grok-4-6-xhigh", "displayModel": "Grok 4.6 (xhigh)", "isUnavailable": false, "variantOf": null, "deepSWE": 64.8968, "terminalBenchV4": 17.6768, "sweAtlasQnA": 58.3333, "meanCostUsdPerTask": 3.5701, "agentWallTimeSec": 1167.3387, "meanSteps": 112.0616, "meanTotalTokens": 5543624.5974, "cacheHitRate": 0.9058, "hostConfigId": "70a5d300021b9986f77d4198272ef1a5", "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } }, "swe-atlas-qna": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } }, "terminal-bench-v4": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-sonnet-5.5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Sonnet 5.5 (medium)", "score": 45.884, "secondary": { "configurationRank": 24, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-medium", "displayModel": "Sonnet 5.5 (medium)", "isUnavailable": false, "variantOf": null, "deepSWE": 65.4867, "terminalBenchV4": 27.2727, "sweAtlasQnA": 44.8925, "meanCostUsdPerTask": 0.6188, "agentWallTimeSec": 510.7722, "meanSteps": 22.209, "meanTotalTokens": 1253351.4928, "cacheHitRate": 0.9341, "hostConfigId": "4d6aa9e9c8df931d54dba5956d94973c", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "qwen3.8-max", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Qwen3.8 Max", "score": 43.2653, "secondary": { "configurationRank": 25, "agentHarness": "Claude Code", "provider": "alibaba_cloud", "hostModelSlug": "alibaba_cloud_qwen3-8-max-public", "displayModel": "Qwen3.8 Max", "isUnavailable": false, "variantOf": null, "deepSWE": 51.0324, "terminalBenchV4": 16.6667, "sweAtlasQnA": 62.0968, "meanCostUsdPerTask": 3.479, "agentWallTimeSec": 3784.4161, "meanSteps": 110.0367, "meanTotalTokens": 8706893.7981, "cacheHitRate": 0.9507, "hostConfigId": "cd4c865dc2f539b650d741fd2ebf7f2d", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.261", "dateReleased": "2026-09-04" }, "max": { "version": "2.1.261", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "2.1.224", "dateReleased": "2026-08-07" }, "max": { "version": "2.1.236", "dateReleased": "2026-08-19" } }, "terminal-bench-v4": { "min": { "version": "2.1.261", "dateReleased": "2026-09-04" }, "max": { "version": "2.1.261", "dateReleased": "2026-09-04" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-5.6-luna", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-5.6 Luna (max)", "score": 43.2247, "secondary": { "configurationRank": 26, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-5-6-luna", "displayModel": "GPT-5.6 Luna (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 66.3717, "terminalBenchV4": 14.6465, "sweAtlasQnA": 48.6559, "meanCostUsdPerTask": 0.438, "agentWallTimeSec": 1412.2609, "meanSteps": 121.5138, "meanTotalTokens": 14854524.5391, "cacheHitRate": 0.9719, "hostConfigId": "b3c56ec69185d544992fd578edb75e8c", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "deepseek-v4-pro", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - DeepSeek V4 Pro 0813 (max)", "score": 43.052, "secondary": { "configurationRank": 27, "agentHarness": "Codex", "provider": "deepseek", "hostModelSlug": "deepseek_deepseek-v4-pro-0813", "displayModel": "DeepSeek V4 Pro 0813 (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 57.2271, "terminalBenchV4": 10.101, "sweAtlasQnA": 61.828, "meanCostUsdPerTask": 0.238, "agentWallTimeSec": 2417.55, "meanSteps": 138.1463, "meanTotalTokens": 24180108.0281, "cacheHitRate": 0.9898, "hostConfigId": "542590ecc1aa12a718d158565d464daa", "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.147.0", "dateReleased": "2026-08-07" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "claude-sonnet-5.5", "boardId": "aa_coding_agent_index", "variantLabel": "Claude Code - Sonnet 5.5 (low)", "score": 42.0593, "secondary": { "configurationRank": 28, "agentHarness": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-low", "displayModel": "Sonnet 5.5 (low)", "isUnavailable": false, "variantOf": null, "deepSWE": 61.9469, "terminalBenchV4": 25.2525, "sweAtlasQnA": 38.9785, "meanCostUsdPerTask": 0.4831, "agentWallTimeSec": 380.8562, "meanSteps": 18.5787, "meanTotalTokens": 977284.7701, "cacheHitRate": 0.9308, "hostConfigId": "e9f9d3ae5b03aa060114dd4b3275d67d", "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gemini-3.8-flash", "boardId": "aa_coding_agent_index", "variantLabel": "Antigravity SDK - Gemini 3.8 Flash (high)", "score": 41.8632, "secondary": { "configurationRank": 29, "agentHarness": "Antigravity SDK v0.1.12", "provider": "google", "hostModelSlug": "google_skimaki_ai-studio", "displayModel": "Gemini 3.8 Flash (high)", "isUnavailable": false, "variantOf": null, "deepSWE": 65.7817, "terminalBenchV4": 14.6465, "sweAtlasQnA": 45.1613, "meanCostUsdPerTask": 2.4674, "agentWallTimeSec": 702.711, "meanSteps": 188.4884, "meanTotalTokens": 13511073.1551, "cacheHitRate": 0.8697, "hostConfigId": "d1003684d8f29b95a3b14d2a7722221a", "versions": { "deep-swe-v1.1": { "min": { "version": "0.1.16", "dateReleased": "2026-09-02" }, "max": { "version": "0.1.16", "dateReleased": "2026-09-02" } }, "swe-atlas-qna": { "min": { "version": "0.1.12", "dateReleased": "2026-08-13" }, "max": { "version": "0.1.12", "dateReleased": "2026-08-13" } }, "terminal-bench-v4": { "min": { "version": "0.1.16", "dateReleased": "2026-09-02" }, "max": { "version": "0.1.16", "dateReleased": "2026-09-02" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "gpt-6-luna", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - GPT-6 Luna (max) ({'reasoning_effort': 'max'})", "score": 41.0744, "secondary": { "configurationRank": 30, "agentHarness": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-6-luna", "displayModel": "GPT-6 Luna (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 63.7168, "terminalBenchV4": 15.1515, "sweAtlasQnA": 44.3548, "meanCostUsdPerTask": 0.1759, "agentWallTimeSec": 1285.9269, "meanSteps": 88.643, "meanTotalTokens": 10218069.2481, "cacheHitRate": 0.9606, "hostConfigId": "ac8276cd5d15ac2daacb9956ad44c132", "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." }, { "model": "deepseek-v4-flash", "boardId": "aa_coding_agent_index", "variantLabel": "Codex - DeepSeek V4 Flash 0731 (max)", "score": 38.7425, "secondary": { "configurationRank": 31, "agentHarness": "Codex", "provider": "deepseek", "hostModelSlug": "deepseek_deepseek-v4-flash-0731", "displayModel": "DeepSeek V4 Flash 0731 (max)", "isUnavailable": false, "variantOf": null, "deepSWE": 54.2773, "terminalBenchV4": 10.6061, "sweAtlasQnA": 51.3441, "meanCostUsdPerTask": 0.0851, "agentWallTimeSec": 1097.0404, "meanSteps": 121.6667, "meanTotalTokens": 15350225.2992, "cacheHitRate": 0.987, "hostConfigId": "28ef65135f08f035dc93a97d324fea73", "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.2", "dateReleased": "2026-09-03" } }, "swe-atlas-qna": { "min": { "version": "0.146.0", "dateReleased": "2026-07-29" }, "max": { "version": "0.146.1", "dateReleased": "2026-08-05" } }, "terminal-bench-v4": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.2", "dateReleased": "2026-09-03" } } } }, "source": "https://artificialanalysis.ai/agents/coding-agents", "checkedAt": "2026-10-01", "note": "Score belongs to this named agent/harness × model configuration; it is not a harness-independent model measurement. indexScore stored as percent (raw fraction × 100)." } ], "unmappedPublicRows": { "aa_coding_agent_index": [ { "displayLabel": "Antigravity CLI - Gemini 4 Argon (high)", "hostModelSlug": "google_barium-bb_ai-studio", "indexScorePercent": 63.7578, "isUnavailable": true, "reason": "model_not_in_adopted:gemini-4-argon" } ], "aa_intelligence_index_notable_ge40": [ { "shortName": "Gemini 4 Argon (high)", "slug": "gemini-4-argon", "score": 52.5606, "estimated": false }, { "shortName": "Claude Opus 4.7 (max)", "slug": "claude-opus-4-7", "score": 40.6898, "estimated": true } ] }, "notFound": [ { "boardId": "aa_intelligence_index", "model": "composer-2.5", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "deepseek-v4-flash-fast", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "deepseek-v4.1-flash-fast", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "glm-5.2-fast", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "glm-5.3-flashx", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "hy4-preview", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "kimi-k2.7-code-highspeed", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "mimo-v2.6-pro-ultraspeed", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "minimax-m3.1-flash-preview", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "muse-spark-1.2-contributor", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "muse-spark-1.3-contributor", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "qwen3.7-flash", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "qwen3.8-flash", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "qwen3.8-omni-flash", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_intelligence_index", "model": "swe-2", "reason": "No exact project-model row in AA models payload 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "claude-fable-5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "claude-opus-4.8", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "claude-sonnet-5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "composer-2.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "deepseek-v4-flash-fast", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "deepseek-v4-flash-vision-exp", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "deepseek-v4.1-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "deepseek-v4.1-flash-fast", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "gemini-3.6-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "gemini-3.7-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "glm-5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "glm-5.1", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "glm-5.2", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "glm-5.2-fast", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "glm-5.3-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "glm-5.3-flashx", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "gpt-5.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "gpt-5.6-terra", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "grok-4.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "hy3", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "hy4-preview", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "inkling", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "inkling-small", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "kimi-k2.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "kimi-k2.6", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "kimi-k2.7-code", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "kimi-k2.7-code-highspeed", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "longcat-2.0", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "mimo-v2.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "mimo-v2.5-pro", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "mimo-v2.6-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "mimo-v2.6-pro", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "mimo-v2.6-pro-ultraspeed", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "minimax-m2.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "minimax-m2.7", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "minimax-m3", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "minimax-m3.1-flash-preview", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "mistral-medium-3.5", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "muse-spark-1.2", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "muse-spark-1.2-contributor", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "muse-spark-1.3-contributor", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "nemotron-3-ultra", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.6-max-preview", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.6-plus", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.7-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.7-max", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.7-plus", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.8-27b", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.8-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.8-max-0902", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "qwen3.8-omni-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "step-3.5-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "step-3.7-flash", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "step-5-preview", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_coding_agent_index", "model": "swe-2", "reason": "No exact project-model configuration among 31 Coding Agent Index v1.5 configurations 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "composer-2.5", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "deepseek-v4-flash-fast", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "deepseek-v4.1-flash-fast", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "glm-5", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "glm-5.2-fast", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "glm-5.3-flashx", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "hy4-preview", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "kimi-k2.5", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "kimi-k2.7-code-highspeed", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "mimo-v2.6-pro-ultraspeed", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "minimax-m2.5", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "minimax-m3.1-flash-preview", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "muse-spark-1.2-contributor", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "muse-spark-1.3-contributor", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "qwen3.6-max-preview", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "qwen3.6-plus", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "qwen3.7-flash", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "qwen3.8-flash", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "qwen3.8-omni-flash", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "step-3.5-flash", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "step-3.7-flash", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." }, { "boardId": "aa_terminal_bench_4", "model": "swe-2", "reason": "No terminalBench40 value for an exactly mapped AA row 2026-10-01." } ], "rawRecords": { "aa_coding_agent_index": [ { "id": "f927d9c41b5c3021996eb14c3dd1341e", "isDefault": true, "isHighlighted": true, "agentName": "Grok Build", "provider": "xai", "hostModelSlug": "xai_grok-bop-o86-xhigh", "display": { "agent": "Grok Build", "model": "Grok 4.7 (xhigh)", "creator": { "agent": "xAI", "model": "xAI" } }, "displayLabel": "Grok Build - Grok 4.7 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 335, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.725663716814159, "inputTokens": 14656743.734513273, "outputTokens": 152948.2005899705 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.629032258064516, "inputTokens": 7966825.954301075, "outputTokens": 57168.37634408602 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 66, "reviewedAttemptCount": 66, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 66 } ], "rate": 0 }, "mean": { "reward": 0.333333333333333, "inputTokens": 25135458.641414143, "outputTokens": 196677.30303030304 } } ], "safety": { "attemptCount": 905, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5626764360706693, "mean": { "reward": 0.5626764360706693, "costUsd": 8.817468730473037, "agentWallTimeSec": 2354.367686468646, "steps": 162.64576457645765, "inputTokens": 14201447.955995597, "outputTokens": 123276.32783278328, "cacheTokens": 13550056.765676552, "cacheHitRate": 0.938882976190201, "totalTokens": 14324724.283828393 }, "sums": { "costUsd": 2671.693025333334 }, "percentiles": { "costUsd": { "p05": 2.7192659427942796, "p25": 4.51524115951595, "p50": 6.599111777777779, "p75": 9.808614987898796, "p95": 18.236931686028605 }, "totalTokens": { "p05": 3709632.12431243, "p25": 6478872.97579758, "p50": 9948394.636963697, "p75": 15872444.876787692, "p95": 31362264.765786577 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } } } }, { "id": "f2de07373c3626f1b9269a6ecad7804c", "isDefault": true, "isHighlighted": true, "agentName": "Muse Code", "provider": "meta", "hostModelSlug": "meta_aa_glacier135", "display": { "agent": "Muse Code", "model": "Muse Spark 1.3 (max)", "creator": { "agent": "Meta", "model": "Meta" } }, "displayLabel": "Muse Code - Muse Spark 1.3 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.716814159292036, "inputTokens": 13526615.77581121, "outputTokens": 122491.69026548673 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.594086021505376, "inputTokens": 4493624.8655913975, "outputTokens": 23615.916666666668 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 63, "reviewedAttemptCount": 63, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 63 } ], "rate": 0 }, "mean": { "reward": 0.318181818181818, "inputTokens": 43900184.35858586, "outputTokens": 955492.9924242424 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5430273329930766, "mean": { "reward": 0.5430273329930766, "costUsd": 3.977504096947204, "agentWallTimeSec": 1106.8533531353135, "steps": 185.23817381738175, "inputTokens": 16445971.068206819, "outputTokens": 263473.5055005501, "cacheTokens": 16090656.487898799, "cacheHitRate": 0.9712826195905685, "totalTokens": 16709444.57370737 }, "sums": { "costUsd": 1205.183741375 }, "percentiles": { "costUsd": { "p05": 0.6994728876952683, "p25": 1.0872893968371842, "p50": 2.0734771829482943, "p75": 3.7535131079207953, "p95": 13.168053740192518 }, "totalTokens": { "p05": 2516743.7783278334, "p25": 4610005.632013202, "p50": 9203495.684268419, "p75": 15732086.408140825, "p95": 62899130.50220029 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } } } }, { "id": "ef63fafa9b4a635522183c879ae8435b", "isDefault": true, "isHighlighted": true, "agentName": "Antigravity CLI", "provider": "google", "hostModelSlug": "google_barium-bb_ai-studio", "display": { "agent": "Antigravity CLI", "model": "Gemini 4 Argon", "creator": { "agent": "Google", "model": "Google" } }, "displayLabel": "Antigravity CLI - Gemini 4 Argon (high)", "variantOf": null, "isUnavailable": true, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 336, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.787610619469027, "inputTokens": 19901657.223214287, "outputTokens": 152422.96130952382 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 371, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.564516129032258, "inputTokens": 7075681.379032258, "outputTokens": 48670.67473118279 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 112, "reviewedAttemptCount": 112, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 112 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.560606060606061, "inputTokens": 15263258.186868686, "outputTokens": 162940.09595959596 } } ], "safety": { "attemptCount": 905, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.637577603035782, "mean": { "reward": 0.637577603035782, "costUsd": 5.8439053894448145, "agentWallTimeSec": 2070.1686032008843, "steps": 149.1832229580574, "inputTokens": 13621672.650110375, "outputTokens": 112121.13134657839, "cacheTokens": 11850855.205596024, "cacheHitRate": 0.8700000000000004, "totalTokens": 13733793.781456953 }, "sums": { "costUsd": 1764.859427612334 }, "percentiles": { "costUsd": { "p05": 1.3165824554546348, "p25": 2.7006862521677717, "p50": 4.259847111864239, "p75": 6.94478139742605, "p95": 15.23551975602052 }, "totalTokens": { "p05": 2407380.2615894047, "p25": 5633674.4067328945, "p50": 9566161.003311247, "p75": 16754521.037527593, "p95": 37940692.606512114 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.2.10", "dateReleased": null }, "max": { "version": "1.2.10", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "1.2.10", "dateReleased": null }, "max": { "version": "1.2.10", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "1.2.10", "dateReleased": null }, "max": { "version": "1.2.10", "dateReleased": null } } } }, { "id": "a2c87c062f3cef73d8525e7578f14742", "isDefault": true, "isHighlighted": true, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-max", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (max)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.71976401179941, "inputTokens": 22525711.609144542, "outputTokens": 614455.6474926254 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 39, "hardStopAttemptCount": 0, "recoveredAttemptCount": 39, "fallbackAttemptCount": 39, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 77, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 10, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 20, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 23, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 29, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 64, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 67, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 97, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 108, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 139, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 147, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 185, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 243, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 254, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 353, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 593, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.10483870967741936 }, "rewardHacking": null, "mean": { "reward": 0.669354838709677, "inputTokens": 34503496.576612905, "outputTokens": 552623.7553763441 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 6, "hardStopAttemptCount": 0, "recoveredAttemptCount": 6, "fallbackAttemptCount": 6, "refusals": [ { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 81, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 87, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 109, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.030303030303030304 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 131, "reviewedAttemptCount": 131, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 131 } ], "rate": 0 }, "mean": { "reward": 0.661616161616162, "inputTokens": 21212057.686868686, "outputTokens": 668705.3535353536 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 45, "hardStopAttemptCount": 0, "recoveredAttemptCount": 45, "fallbackAttemptCount": 45, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 77, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 10, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 20, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 23, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 29, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 64, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 67, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 97, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 108, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 139, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 147, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 185, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 243, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 254, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 353, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 593, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 81, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 87, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 109, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.045047246660149885 }, "indexScore": 0.6835783373750831, "mean": { "reward": 0.6835783373750831, "costUsd": 14.190800796369649, "agentWallTimeSec": 5244.965372937293, "steps": 265.93069306930687, "inputTokens": 27141368.959295932, "outputTokens": 600968.2744774478, "cacheTokens": 26006027.66886688, "cacheHitRate": 0.9506214943317096, "totalTokens": 27742337.233773403 }, "sums": { "costUsd": 4299.812641299994 }, "percentiles": { "costUsd": { "p05": 4.66515082839384, "p25": 7.936658873542355, "p50": 12.34840697040706, "p75": 19.223640478410346, "p95": 28.29229594782175 }, "totalTokens": { "p05": 6340397.848074809, "p25": 13743009.44829484, "p50": 23752423.691969212, "p75": 37935915.66336632, "p95": 59829310.28954897 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "9e810ff3c6cce5ebaaff7c744c687a47", "isDefault": true, "isHighlighted": true, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_vega-alpha", "display": { "agent": "Codex", "model": "GPT-6 Astra (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6 Astra (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.675516224188791, "inputTokens": 3338101.7050147494, "outputTokens": 56420.18289085546 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 19, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 8, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 16, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03494623655913978 }, "rewardHacking": null, "mean": { "reward": 0.618279569892473, "inputTokens": 2623791.1885245903, "outputTokens": 28534.494535519127 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 110, "reviewedAttemptCount": 110, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 110 } ], "rate": 0 }, "mean": { "reward": 0.555555555555555, "inputTokens": 4503151.8838383835, "outputTokens": 65844.76262626263 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 19, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 8, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 16, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.01164874551971326 }, "indexScore": 0.6164504498789397, "mean": { "reward": 0.6164504498789397, "costUsd": 7.46709735138597, "agentWallTimeSec": 1762.0330264026397, "steps": 39.378437843784376, "inputTokens": 3299549.915435806, "outputTokens": 47061.09677361179, "cacheTokens": 3141588.9050659174, "cacheHitRate": 0.9335591564904552, "totalTokens": 3346611.012209417 }, "sums": { "costUsd": 2251.127294666667 }, "percentiles": { "costUsd": { "p05": 2.7390315410891106, "p25": 4.340413036578659, "p50": 6.125789602860287, "p75": 8.728907821232124, "p95": 14.871133791914197 }, "totalTokens": { "p05": 824190.9740374036, "p25": 1550558.2557755776, "p50": 2403033.1683168346, "p75": 3942392.7057205727, "p95": 7480476.901430149 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.151.0", "dateReleased": "2026-08-29" }, "max": { "version": "0.151.0", "dateReleased": "2026-08-29" } }, "terminal-bench-v4": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, { "id": "29a1cdad0d140781d05b9b4746cfdd0d", "isDefault": true, "isHighlighted": true, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-fable-5-1", "display": { "agent": "Claude Code", "model": "Fable 5.1 (max) (with fallback)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Fable 5.1 (max) (with fallback)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.64306784660767, "inputTokens": 6428754.772861357, "outputTokens": 155542.35398230088 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 46, "hardStopAttemptCount": 0, "recoveredAttemptCount": 46, "fallbackAttemptCount": 46, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 31, "outcome": "fallback", "rate": 0.08333333333333333 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 40, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 112, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.12365591397849462 }, "rewardHacking": null, "mean": { "reward": 0.647849462365591, "inputTokens": 3623187.8387096776, "outputTokens": 81143.10752688172 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 28, "hardStopAttemptCount": 0, "recoveredAttemptCount": 28, "fallbackAttemptCount": 28, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.030303030303030304 }, { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.010101010101010102 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 5, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 31, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.1414141414141414 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 114, "reviewedAttemptCount": 114, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 114 } ], "rate": 0 }, "mean": { "reward": 0.575757575757576, "inputTokens": 7810711.565656566, "outputTokens": 194732.47474747474 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 74, "hardStopAttemptCount": 0, "recoveredAttemptCount": 74, "fallbackAttemptCount": 74, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 31, "outcome": "fallback", "rate": 0.027777777777777776 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 40, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 112, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.010101010101010102 }, { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.0033670033670033673 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 5, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 31, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.08835668513087869 }, "indexScore": 0.6222249615769457, "mean": { "reward": 0.6222249615769457, "costUsd": 12.388092375137534, "agentWallTimeSec": 2089.900125412543, "steps": 36.65676567656766, "inputTokens": 5581622.259625964, "outputTokens": 133631.59955995603, "cacheTokens": 5205555.651265128, "cacheHitRate": 0.9159555838702423, "totalTokens": 5715253.85918592 }, "sums": { "costUsd": 3753.591989666663 }, "percentiles": { "costUsd": { "p05": 5.444339298624863, "p25": 7.8804961398515, "p50": 11.040917095709577, "p75": 14.93645990085259, "p95": 22.439563513833875 }, "totalTokens": { "p05": 1605004.4331133126, "p25": 2728342.3767876793, "p50": 4432038.257425743, "p75": 7298906.072607261, "p95": 13976824.222882291 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } }, "swe-atlas-qna": { "min": { "version": "2.1.259", "dateReleased": "2026-09-02" }, "max": { "version": "2.1.259", "dateReleased": "2026-09-02" } }, "terminal-bench-v4": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } } } }, { "id": "ab5074e404be48bbd4ac3c488ad9a3e2", "isDefault": true, "isHighlighted": true, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-opus-5-5", "display": { "agent": "Claude Code", "model": "Opus 5.5 (max)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Opus 5.5 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 2, "hardStopAttemptCount": 0, "recoveredAttemptCount": 2, "fallbackAttemptCount": 2, "refusals": [ { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 } ], "rate": 0.0058997050147492625 }, "rewardHacking": null, "mean": { "reward": 0.684365781710915, "inputTokens": 15895334.887905605, "outputTokens": 406479.005899705 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 52, "hardStopAttemptCount": 0, "recoveredAttemptCount": 52, "fallbackAttemptCount": 52, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 25, "outcome": "fallback", "rate": 0.06720430107526881 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 41, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 54, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 63, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 71, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 72, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 82, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 98, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 102, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 127, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 132, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 241, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 261, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 386, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.13978494623655913 }, "rewardHacking": null, "mean": { "reward": 0.663978494623656, "inputTokens": 14842420.086021505, "outputTokens": 214788.37634408602 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 24, "hardStopAttemptCount": 0, "recoveredAttemptCount": 24, "fallbackAttemptCount": 24, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.010101010101010102 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 44, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 58, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 88, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 126, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 360, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 436, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.12121212121212122 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 125, "reviewedAttemptCount": 125, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 125 } ], "rate": 0 }, "mean": { "reward": 0.631313131313131, "inputTokens": 14764903.207070706, "outputTokens": 430101.28282828286 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 78, "hardStopAttemptCount": 0, "recoveredAttemptCount": 78, "fallbackAttemptCount": 78, "refusals": [ { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 25, "outcome": "fallback", "rate": 0.022401433691756272 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 41, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 54, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 63, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 71, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 72, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 82, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 98, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 102, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 127, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 132, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 241, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 261, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 386, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.0033670033670033673 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 44, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 58, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 88, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 126, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 360, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 436, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.08896559082114319 }, "indexScore": 0.659885802549234, "mean": { "reward": 0.659885802549234, "costUsd": 13.036383418426828, "agentWallTimeSec": 3867.0923410341024, "steps": 155.0957095709571, "inputTokens": 15218206.418041805, "outputTokens": 333176.801980198, "cacheTokens": 14568923.550055025, "cacheHitRate": 0.9500106599458848, "totalTokens": 15551383.220022004 }, "sums": { "costUsd": 3950.02417578334 }, "percentiles": { "costUsd": { "p05": 4.819416574015403, "p25": 7.233748635478535, "p50": 11.484682617381733, "p75": 16.62301155423542, "p95": 27.619909799086923 }, "totalTokens": { "p05": 3916930.348074808, "p25": 7542033.144664456, "p50": 12988730.595159527, "p75": 20374948.502200224, "p95": 34511194.44961495 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "ed8c1d0b0a2fe4560bbfa0a9f47e8941", "isDefault": true, "isHighlighted": true, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (xhigh)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (xhigh) ({'reasoning_effort': 'xhigh'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.731563421828909, "inputTokens": 2690171.784660767, "outputTokens": 37762.082595870204 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 18, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 20, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 27, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03494623655913978 }, "rewardHacking": null, "mean": { "reward": 0.610215053763441, "inputTokens": 2748674.931693989, "outputTokens": 21382.483606557376 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 109, "reviewedAttemptCount": 109, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 109 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.545454545454545, "inputTokens": 4815817, "outputTokens": 57703.767676767675 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 18, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 20, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 27, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.01164874551971326 }, "indexScore": 0.6290776736822983, "mean": { "reward": 0.6290776736822983, "costUsd": 1.0397081014409635, "agentWallTimeSec": 931.3823091309132, "steps": 40.05500550055005, "inputTokens": 3177125.49569875, "outputTokens": 35402.61375317859, "cacheTokens": 3036065.628757956, "cacheHitRate": 0.9327572280083256, "totalTokens": 3212528.1094519314 }, "sums": { "costUsd": 313.4238835999993 }, "percentiles": { "costUsd": { "p05": 0.39666282002200215, "p25": 0.5709024621012102, "p50": 0.818101308030803, "p75": 1.1004568787128721, "p95": 2.1570592566556637 }, "totalTokens": { "p05": 682378.7798679869, "p25": 1314619.7282728287, "p50": 2203242.6336633656, "p75": 3691119.964796481, "p95": 7871107.464686463 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "cd60470c35fd626a116af80fd5915dfa", "isDefault": false, "isHighlighted": false, "agentName": "Muse Code 1.0.2 RC", "provider": "meta", "hostModelSlug": "meta_joyful-jello138", "display": { "agent": "Muse Code", "model": "Muse Spark 1.3 (xhigh)", "creator": { "agent": "Meta", "model": "Meta" } }, "displayLabel": "Muse Code - Muse Spark 1.3 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.731563421828909, "inputTokens": 15210161.250737464, "outputTokens": 78929.04719764012 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.545698924731183, "inputTokens": 3764504.801075269, "outputTokens": 21519.38440860215 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 35, "reviewedAttemptCount": 35, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 35 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.171717171717172, "inputTokens": 40760107.72474747, "outputTokens": 493004.18434343435 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.482993172759088, "mean": { "reward": 0.482993172759088, "costUsd": 3.467754041474145, "agentWallTimeSec": 1098.202911991198, "steps": 187.48954895489547, "inputTokens": 16091465.103960397, "outputTokens": 145629.2480748075, "cacheTokens": 15709288.80143016, "cacheHitRate": 0.9661832360411203, "totalTokens": 16237094.352035202 }, "sums": { "costUsd": 1050.7294745666659 }, "percentiles": { "costUsd": { "p05": 0.6543826890539068, "p25": 1.1450520823432337, "p50": 1.6800104498349842, "p75": 3.3331835402640264, "p95": 11.812090951716167 }, "totalTokens": { "p05": 2805937.5105610583, "p25": 5191748.90649065, "p50": 8172270.941694162, "p75": 15773437.224972516, "p95": 58864615.07931783 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } }, "swe-atlas-qna": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } }, "terminal-bench-v4": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } } } }, { "id": "28ef65135f08f035dc93a97d324fea73", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "deepseek", "hostModelSlug": "deepseek_deepseek-v4-flash-0731", "display": { "agent": "Codex", "model": "DeepSeek V4 Flash 0731 (max)", "creator": { "agent": "OpenAI", "model": "DeepSeek" } }, "displayLabel": "Codex - DeepSeek V4 Flash 0731 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.542772861356932, "inputTokens": 18697668.702064898, "outputTokens": 97092.21238938053 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.513440860215054, "inputTokens": 8160643.36827957, "outputTokens": 35348.784946236556 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 21, "reviewedAttemptCount": 21, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 21 } ], "rate": 0 }, "mean": { "reward": 0.106060606060606, "inputTokens": 22757591.373737372, "outputTokens": 136457.94949494948 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.3874247758775307, "mean": { "reward": 0.3874247758775307, "costUsd": 0.08506419165060494, "agentWallTimeSec": 1097.0404400440034, "steps": 121.66666666666667, "inputTokens": 15269826.30913091, "outputTokens": 80398.9900990099, "cacheTokens": 15142136.818481851, "cacheHitRate": 0.9869993314104538, "totalTokens": 15350225.299229927 }, "sums": { "costUsd": 25.774450070133362 }, "percentiles": { "costUsd": { "p05": 0.034830933659581974, "p25": 0.054086775420022014, "p50": 0.07488192848580855, "p75": 0.09969517195863582, "p95": 0.17027440660206825 }, "totalTokens": { "p05": 3870194.438283829, "p25": 7621129.167766776, "p50": 11973775.719471961, "p75": 19587023.848184794, "p95": 38473133.53927391 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.2", "dateReleased": "2026-09-03" } }, "swe-atlas-qna": { "min": { "version": "0.146.0", "dateReleased": "2026-07-29" }, "max": { "version": "0.146.1", "dateReleased": "2026-08-05" } }, "terminal-bench-v4": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.2", "dateReleased": "2026-09-03" } } } }, { "id": "33e80f383dc644527161435b076798ad", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-high", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (high)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (high)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.666666666666667, "inputTokens": 2606937.696165192, "outputTokens": 46314.92330383481 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 31, "hardStopAttemptCount": 0, "recoveredAttemptCount": 31, "fallbackAttemptCount": 31, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.013440860215053764 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 37, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 83, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 96, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.08333333333333333 }, "rewardHacking": null, "mean": { "reward": 0.564516129032258, "inputTokens": 1796491.9838709678, "outputTokens": 20475.11559139785 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 2, "hardStopAttemptCount": 0, "recoveredAttemptCount": 2, "fallbackAttemptCount": 2, "refusals": [ { "turn": 92, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 128, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.010101010101010102 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 83, "reviewedAttemptCount": 83, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 83 } ], "rate": 0 }, "mean": { "reward": 0.419191919191919, "inputTokens": 4147711.272727273, "outputTokens": 72740.92424242424 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 33, "hardStopAttemptCount": 0, "recoveredAttemptCount": 33, "fallbackAttemptCount": 33, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.004480286738351254 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 37, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 83, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 96, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 92, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 128, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.031144781144781142 }, "indexScore": 0.5501249049636148, "mean": { "reward": 0.5501249049636148, "costUsd": 1.235497127172719, "agentWallTimeSec": 735.1809878987912, "steps": 36.772277227722775, "inputTokens": 2610884.190319032, "outputTokens": 41496.37513751376, "cacheTokens": 2507283.205720573, "cacheHitRate": 0.9477934370079523, "totalTokens": 2652380.565456549 }, "sums": { "costUsd": 374.355629533333 }, "percentiles": { "costUsd": { "p05": 0.26541014830583043, "p25": 0.45273301105610575, "p50": 0.8423270638063807, "p75": 1.452190312926292, "p95": 3.3796598031683143 }, "totalTokens": { "p05": 393734.5768976899, "p25": 771180.2568756866, "p50": 1799032.7986798703, "p75": 3205427.8019801965, "p95": 8253061.206600648 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "666605c66ad569c503ce53c01682e285", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-xhigh", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (xhigh)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.684365781710914, "inputTokens": 7430693.578171091, "outputTokens": 150014.38643067845 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 31, "hardStopAttemptCount": 0, "recoveredAttemptCount": 31, "fallbackAttemptCount": 31, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 115, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 162, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.08333333333333333 }, "rewardHacking": null, "mean": { "reward": 0.620967741935484, "inputTokens": 5431788.322580645, "outputTokens": 64841.01344086022 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 65, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.015151515151515152 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 115, "reviewedAttemptCount": 115, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 115 } ], "rate": 0 }, "mean": { "reward": 0.580808080808081, "inputTokens": 9034401.93939394, "outputTokens": 198393.7626262626 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 34, "hardStopAttemptCount": 0, "recoveredAttemptCount": 34, "fallbackAttemptCount": 34, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 115, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 162, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 65, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.032828282828282825 }, "indexScore": 0.628713868151493, "mean": { "reward": 0.628713868151493, "costUsd": 3.3333865348734886, "agentWallTimeSec": 1619.0134708470848, "steps": 78.31683168316832, "inputTokens": 6961982.357535753, "outputTokens": 125696.03850385037, "cacheTokens": 6693763.679867986, "cacheHitRate": 0.9574714756466342, "totalTokens": 7087678.3960396005 }, "sums": { "costUsd": 1010.0161200666669 }, "percentiles": { "costUsd": { "p05": 1.013542773971397, "p25": 1.7970568723322353, "p50": 2.8112455694169425, "p75": 4.45485277458746, "p95": 7.184486563465355 }, "totalTokens": { "p05": 1669169.926072607, "p25": 3466916.930693066, "p50": 5558230.887788775, "p75": 9190033.857535753, "p95": 17286164.792959306 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "ac8276cd5d15ac2daacb9956ad44c132", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-6-luna", "display": { "agent": "Codex", "model": "GPT-6 Luna (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6 Luna (max) ({'reasoning_effort': 'max'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 338, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.63716814159292, "inputTokens": 12385746.131268436, "outputTokens": 108271.5634218289 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.443548387096774, "inputTokens": 5786509.854838709, "outputTokens": 53341.779569892475 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 30, "reviewedAttemptCount": 30, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 30 } ], "rate": 0 }, "mean": { "reward": 0.151515151515152, "inputTokens": 14381227.393939395, "outputTokens": 165884.42424242425 } } ], "safety": { "attemptCount": 908, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4107438934016153, "mean": { "reward": 0.4107438934016153, "costUsd": 0.17591172304455457, "agentWallTimeSec": 1285.9268514851478, "steps": 88.64301430143014, "inputTokens": 10119727.864136413, "outputTokens": 98341.38393839383, "cacheTokens": 9897312.69911991, "cacheHitRate": 0.9606438143896164, "totalTokens": 10218069.248074815 }, "sums": { "costUsd": 53.3012520824999 }, "percentiles": { "costUsd": { "p05": 0.05139677199229924, "p25": 0.08709199411441132, "p50": 0.12790623996149636, "p75": 0.23731735201595155, "p95": 0.4026450882722773 }, "totalTokens": { "p05": 2000123.225962595, "p25": 3915894.559955995, "p50": 7023106.606160604, "p75": 13626608.82783279, "p95": 26835833.369636964 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "d1003684d8f29b95a3b14d2a7722221a", "isDefault": false, "isHighlighted": false, "agentName": "Antigravity SDK v0.1.12", "provider": "google", "hostModelSlug": "google_skimaki_ai-studio", "display": { "agent": "Antigravity SDK", "model": "Gemini 3.8 Flash (high)", "creator": { "agent": "Google", "model": "Google" } }, "displayLabel": "Antigravity SDK - Gemini 3.8 Flash (high)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.657817109144543, "inputTokens": 16400687.365781711, "outputTokens": 148259.0501474926 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.451612903225807, "inputTokens": 9020406.623655913, "outputTokens": 58109 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 33, "reviewedAttemptCount": 33, "flaggedAttemptCount": 4, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 33 } ], "rate": 0.020202020202020204 }, "mean": { "reward": 0.146464646464646, "inputTokens": 16468964.714646464, "outputTokens": 168736.55303030304 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4186315529449987, "mean": { "reward": 0.4186315529449987, "costUsd": 2.4673922004125437, "agentWallTimeSec": 702.7110253025301, "steps": 188.4884488448845, "inputTokens": 13395246.748624865, "outputTokens": 115826.40649064905, "cacheTokens": 11871691.682068229, "cacheHitRate": 0.8697428234581179, "totalTokens": 13511073.155115515 }, "sums": { "costUsd": 747.619836725 }, "percentiles": { "costUsd": { "p05": 1.099103153622112, "p25": 1.6439984646864687, "p50": 2.1229609128712874, "p75": 3.008778207632013, "p95": 4.849891568358086 }, "totalTokens": { "p05": 4679184.810451044, "p25": 8015173.094609471, "p50": 11559537.268426849, "p75": 16883320.984598454, "p95": 28318875.752915297 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.1.16", "dateReleased": "2026-09-02" }, "max": { "version": "0.1.16", "dateReleased": "2026-09-02" } }, "swe-atlas-qna": { "min": { "version": "0.1.12", "dateReleased": "2026-08-13" }, "max": { "version": "0.1.12", "dateReleased": "2026-08-13" } }, "terminal-bench-v4": { "min": { "version": "0.1.16", "dateReleased": "2026-09-02" }, "max": { "version": "0.1.16", "dateReleased": "2026-09-02" } } } }, { "id": "fa2595093fa640e73d072e7d1aa7df82", "isDefault": true, "isHighlighted": false, "agentName": "Devin Fusion CLI", "provider": "cognition", "hostModelSlug": "cognition_fusion-gpt-6-astra-xhigh-sidekick-penguin-medium", "display": { "agent": "Devin Fusion CLI", "model": "GPT-6 Astra XHigh + SWE-2 Medium" }, "displayLabel": "Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 1, "hardStopAttemptCount": 0, "recoveredAttemptCount": 1, "fallbackAttemptCount": 1, "refusals": [ { "turn": 5, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 } ], "rate": 0.0029498525073746312 }, "rewardHacking": null, "mean": { "reward": 0.672566371681416, "inputTokens": 10312445.209439527, "outputTokens": 83838.11799410029 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 25, "hardStopAttemptCount": 1, "recoveredAttemptCount": 24, "fallbackAttemptCount": 24, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.013440860215053764 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 1, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 10, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.06720430107526881 }, "rewardHacking": null, "mean": { "reward": 0.594086021505376, "inputTokens": 2112115.313172043, "outputTokens": 23623.837365591397 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 99, "reviewedAttemptCount": 99, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 99 } ], "rate": 0 }, "mean": { "reward": 0.5, "inputTokens": 6001673.575757576, "outputTokens": 71827.65656565657 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 26, "hardStopAttemptCount": 1, "recoveredAttemptCount": 25, "fallbackAttemptCount": 25, "refusals": [ { "turn": 5, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.004480286738351254 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 1, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 10, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.023384717860881147 }, "indexScore": 0.588884131062264, "mean": { "reward": 0.588884131062264, "costUsd": 4.536492777821779, "agentWallTimeSec": 1481.89598019802, "steps": 78.55115511551155, "inputTokens": 6017554.665016501, "outputTokens": 56579.83003300329, "cacheTokens": 5695796.99229923, "cacheHitRate": 0.920061132172241, "totalTokens": 6074134.4950494915 }, "sums": { "costUsd": 1374.557311679999 }, "percentiles": { "costUsd": { "p05": 1.8884645795379544, "p25": 2.7901938516089118, "p50": 3.6622604586358625, "p75": 5.514399754826733, "p95": 9.903196057832783 }, "totalTokens": { "p05": 1278837.6278327832, "p25": 2419160.8525852584, "p50": 3840277.500550057, "p75": 7328820.564356423, "p95": 15347920.93509351 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } } } }, { "id": "cd4c865dc2f539b650d741fd2ebf7f2d", "isDefault": true, "isHighlighted": false, "agentName": "Claude Code", "provider": "alibaba_cloud", "hostModelSlug": "alibaba_cloud_qwen3-8-max-public", "display": { "agent": "Claude Code", "model": "Qwen3.8 Max", "creator": { "agent": "Anthropic", "model": "Alibaba Cloud" } }, "displayLabel": "Claude Code - Qwen3.8 Max", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.510324483775811, "inputTokens": 10307762.331858408, "outputTokens": 104418.39233038347 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.620967741935484, "inputTokens": 5159227.653225807, "outputTokens": 45811.73655913978 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 33, "reviewedAttemptCount": 33, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 33 } ], "rate": 0 }, "mean": { "reward": 0.166666666666667, "inputTokens": 12143161.925641026, "outputTokens": 223315.56923076924 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.43265296412598736, "mean": { "reward": 0.43265296412598736, "costUsd": 3.478965577837015, "agentWallTimeSec": 3784.416073707372, "steps": 110.03670136244394, "inputTokens": 8600561.252779894, "outputTokens": 106332.54533299482, "cacheTokens": 8290829.75631717, "cacheHitRate": 0.9507113544554301, "totalTokens": 8706893.798112892 }, "sums": { "costUsd": 1048.213634833334 }, "percentiles": { "costUsd": { "p05": 1.2462484540154002, "p25": 1.9738426430143006, "p50": 2.838224532178221, "p75": 4.649543275302529, "p95": 7.302521150137508 }, "totalTokens": { "p05": 2586103.0433443333, "p25": 4938223.044004401, "p50": 7296928.9218921885, "p75": 11298465.588558838, "p95": 18720391.11265126 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.261", "dateReleased": "2026-09-04" }, "max": { "version": "2.1.261", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "2.1.224", "dateReleased": "2026-08-07" }, "max": { "version": "2.1.236", "dateReleased": "2026-08-19" } }, "terminal-bench-v4": { "min": { "version": "2.1.261", "dateReleased": "2026-09-04" }, "max": { "version": "2.1.261", "dateReleased": "2026-09-04" } } } }, { "id": "34970c963bdfd318e46205514d46ea4f", "isDefault": true, "isHighlighted": false, "agentName": "Devin Fusion CLI", "provider": "cognition", "hostModelSlug": "cognition_fusion-claude-fable-5-1-xhigh-sidekick-penguin-medium", "display": { "agent": "Devin Fusion CLI", "model": "Claude Fable 5.1 XHigh + SWE-2 Medium" }, "displayLabel": "Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 1, "hardStopAttemptCount": 0, "recoveredAttemptCount": 1, "fallbackAttemptCount": 1, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 } ], "rate": 0.0029498525073746312 }, "rewardHacking": null, "mean": { "reward": 0.631268436578171, "inputTokens": 10080280.292035399, "outputTokens": 111599.9557522124 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 42, "hardStopAttemptCount": 0, "recoveredAttemptCount": 42, "fallbackAttemptCount": 42, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 21, "outcome": "fallback", "rate": 0.056451612903225805 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 22, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 36, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 37, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 60, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.11290322580645161 }, "rewardHacking": null, "mean": { "reward": 0.658602150537635, "inputTokens": 5492119.514784946, "outputTokens": 55827.5685483871 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 197, "refusedAttemptCount": 19, "hardStopAttemptCount": 0, "recoveredAttemptCount": 19, "fallbackAttemptCount": 19, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.030456852791878174 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.01015228426395939 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 6, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 8, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 13, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 25, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 } ], "rate": 0.09644670050761421 }, "rewardHacking": { "retainedAttemptCount": 197, "passingAttemptCount": 111, "reviewedAttemptCount": 111, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 111 } ], "rate": 0 }, "mean": { "reward": 0.560606060606061, "inputTokens": 16451944.767676767, "outputTokens": 167921.75252525252 } } ], "safety": { "attemptCount": 908, "refusedAttemptCount": 62, "hardStopAttemptCount": 0, "recoveredAttemptCount": 62, "fallbackAttemptCount": 62, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 21, "outcome": "fallback", "rate": 0.01881720430107527 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 22, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 36, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 37, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 60, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.010152284263959392 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.00338409475465313 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 6, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 8, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 13, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 25, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 } ], "rate": 0.07076659294048016 }, "indexScore": 0.6168255492406223, "mean": { "reward": 0.6168255492406223, "costUsd": 7.897126985217273, "agentWallTimeSec": 2146.9920896589642, "steps": 99.53135313531354, "inputTokens": 9590504.44719472, "outputTokens": 101043.72662266226, "cacheTokens": 9279592.760176014, "cacheHitRate": 0.9584900963367992, "totalTokens": 9691548.173817372 }, "sums": { "costUsd": 2392.829476520833 }, "percentiles": { "costUsd": { "p05": 3.1036853524587458, "p25": 4.744201155569304, "p50": 6.569551072249726, "p75": 9.734336789150163, "p95": 16.72564247919966 }, "totalTokens": { "p05": 1840241.9694169401, "p25": 3708383.8074807483, "p50": 6571844.10121012, "p75": 12694519.213971417, "p95": 27823135.185038492 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } } } }, { "id": "709f801275dedae5675b1f56e97c9df3", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-5-6-sol", "display": { "agent": "Codex", "model": "GPT-5.6 Sol (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-5.6 Sol (max) ({'reasoning_effort': 'max'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.722713864306785, "inputTokens": 10907777.09439528, "outputTokens": 56943.890855457226 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 1, "hardStopAttemptCount": 1, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.002688172043010753 }, "rewardHacking": null, "mean": { "reward": 0.540322580645161, "inputTokens": 7898018.697580645, "outputTokens": 31499.083333333332 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 74, "reviewedAttemptCount": 74, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 74 } ], "rate": 0 }, "mean": { "reward": 0.373737373737374, "inputTokens": 12950366.515151516, "outputTokens": 68992.93181818182 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 1, "hardStopAttemptCount": 1, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.000896057347670251 }, "indexScore": 0.54559127289644, "mean": { "reward": 0.54559127289644, "costUsd": 6.353318230693071, "agentWallTimeSec": 1237.696445544555, "steps": 116.39493949394938, "inputTokens": 10120981.254675467, "outputTokens": 49155.37788778878, "cacheTokens": 9784083.974697473, "cacheHitRate": 0.9458037017976718, "totalTokens": 10170136.632563243 }, "sums": { "costUsd": 1925.0554239 }, "percentiles": { "costUsd": { "p05": 1.8512760725412558, "p25": 3.1885388294829466, "p50": 5.214009559955996, "p75": 7.762229316171614, "p95": 14.896605548536868 }, "totalTokens": { "p05": 1994550.4016501652, "p25": 3937210.7585258516, "p50": 7714596.468646868, "p75": 12446152.79537957, "p95": 27006730.063476328 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.139.0", "dateReleased": "2026-06-09" }, "max": { "version": "0.147.0", "dateReleased": "2026-08-07" } }, "terminal-bench-v4": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, { "id": "93acd59f31f8e5478686d650aa650f79", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (medium)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (medium) ({'reasoning_effort': 'medium'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.71976401179941, "inputTokens": 1875718.3569321535, "outputTokens": 22954.34808259587 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 12, "hardStopAttemptCount": 12, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 10, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03225806451612903 }, "rewardHacking": null, "mean": { "reward": 0.60752688172043, "inputTokens": 1970188.7650273223, "outputTokens": 13754.166666666666 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 102, "reviewedAttemptCount": 102, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 102 } ], "rate": 0 }, "mean": { "reward": 0.515151515151515, "inputTokens": 3535272.3131313133, "outputTokens": 37750.72727272727 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 12, "hardStopAttemptCount": 12, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 10, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.01075268817204301 }, "indexScore": 0.6141474695571183, "mean": { "reward": 0.6141474695571183, "costUsd": 0.7046746644898908, "agentWallTimeSec": 651.3228668866898, "steps": 37.98899889988999, "inputTokens": 2275866.5144006205, "outputTokens": 22412.23102310231, "cacheTokens": 2176436.9277894986, "cacheHitRate": 0.9239660644436648, "totalTokens": 2298278.745423723 }, "sums": { "costUsd": 212.41134426666656 }, "percentiles": { "costUsd": { "p05": 0.21984839405940593, "p25": 0.3595459733773378, "p50": 0.4937551212321231, "p75": 0.7913959203520352, "p95": 1.468924121518151 }, "totalTokens": { "p05": 376691.5391639165, "p25": 783633.2106710669, "p50": 1298877.5742574276, "p75": 2736930.987898792, "p95": 5339040.968976896 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "ea296ce785422d3b97cfff4427f2e0d2", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.696165191740413, "inputTokens": 3676096.7050147494, "outputTokens": 70758.95280235988 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 15, "hardStopAttemptCount": 15, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 25, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 23, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 32, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 65, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.04032258064516129 }, "rewardHacking": null, "mean": { "reward": 0.577956989247312, "inputTokens": 3171478.0546448086, "outputTokens": 35668.81420765027 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 105, "reviewedAttemptCount": 105, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 105 } ], "rate": 0 }, "mean": { "reward": 0.53030303030303, "inputTokens": 5825821.813131313, "outputTokens": 94906.77777777778 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 15, "hardStopAttemptCount": 15, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 25, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 23, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 32, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 65, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.013440860215053764 }, "indexScore": 0.6014750704302517, "mean": { "reward": 0.6014750704302517, "costUsd": 1.554119528390054, "agentWallTimeSec": 1463.0464224422437, "steps": 40.45654565456545, "inputTokens": 3937843.0564663024, "outputTokens": 61658.55432920342, "cacheTokens": 3740552.1873072563, "cacheHitRate": 0.9281954641245814, "totalTokens": 3999501.6107955067 }, "sums": { "costUsd": 468.718937949999 }, "percentiles": { "costUsd": { "p05": 0.50620098809681, "p25": 0.7844812155115526, "p50": 1.140491010561054, "p75": 1.8937155413641367, "p95": 3.7431764045654536 }, "totalTokens": { "p05": 795880.5675467533, "p25": 1648548.6529152917, "p50": 2724214.3113311334, "p75": 4694077.841584162, "p95": 10822464.925852576 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "409518818f1e9ad21995d70631ab05b6", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-opus-5", "display": { "agent": "Claude Code", "model": "Opus 5 (max)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Opus 5 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.625368731563422, "inputTokens": 13358102.041297935, "outputTokens": 129793.94690265486 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.620967741935484, "inputTokens": 7586223.970430108, "outputTokens": 61960.48924731183 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 197, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 59, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 } ], "rate": 0.015228426395939087 }, "rewardHacking": { "retainedAttemptCount": 197, "passingAttemptCount": 108, "reviewedAttemptCount": 108, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 108 } ], "rate": 0 }, "mean": { "reward": 0.545454545454545, "inputTokens": 14469933.381313132, "outputTokens": 288742.0505050505 } } ], "safety": { "attemptCount": 908, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 59, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 } ], "rate": 0.005076142131979696 }, "indexScore": 0.5972636729844837, "mean": { "reward": 0.5972636729844837, "costUsd": 10.789154699257413, "agentWallTimeSec": 2515.540235973597, "steps": 152.13036303630363, "inputTokens": 11238194.40979098, "outputTokens": 136656.0792079208, "cacheTokens": 10929014.04565457, "cacheHitRate": 0.9698258997320075, "totalTokens": 11374850.488998907 }, "sums": { "costUsd": 3269.1138738750005 }, "percentiles": { "costUsd": { "p05": 4.070634550165014, "p25": 6.0050860609185905, "p50": 8.377338476347639, "p75": 13.669421935368524, "p95": 24.306012511496142 }, "totalTokens": { "p05": 3637409.260286029, "p25": 6155919.92079208, "p50": 9436168.620462047, "p75": 14396226.760176001, "p95": 24864036.97546756 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } }, "swe-atlas-qna": { "min": { "version": "2.1.218", "dateReleased": "2026-07-22" }, "max": { "version": "2.1.227", "dateReleased": "2026-08-10" } }, "terminal-bench-v4": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } } } }, { "id": "e9f9d3ae5b03aa060114dd4b3275d67d", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-low", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (low)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (low)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.619469026548673, "inputTokens": 831380.2064896756, "outputTokens": 16103.77581120944 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 15, "hardStopAttemptCount": 0, "recoveredAttemptCount": 15, "fallbackAttemptCount": 15, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 4, "outcome": "fallback", "rate": 0.010752688172043012 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.04032258064516129 }, "rewardHacking": null, "mean": { "reward": 0.389784946236559, "inputTokens": 393558.31720430107, "outputTokens": 6681.293010752688 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 138, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 183, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 365, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.015151515151515152 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 50, "reviewedAttemptCount": 50, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 50 } ], "rate": 0 }, "mean": { "reward": 0.252525252525253, "inputTokens": 2251236.696969697, "outputTokens": 32428.20707070707 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 18, "hardStopAttemptCount": 0, "recoveredAttemptCount": 18, "fallbackAttemptCount": 18, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 4, "outcome": "fallback", "rate": 0.003584229390681004 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 138, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 183, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 365, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.018491365265558814 }, "indexScore": 0.420593075103495, "mean": { "reward": 0.420593075103495, "costUsd": 0.48311011474147336, "agentWallTimeSec": 380.85617931793183, "steps": 18.578657865786578, "inputTokens": 961481.2431243125, "outputTokens": 15803.526952695267, "cacheTokens": 917182.5137513754, "cacheHitRate": 0.9307814644010347, "totalTokens": 977284.7700770069 }, "sums": { "costUsd": 146.3823647666666 }, "percentiles": { "costUsd": { "p05": 0.10326793619361947, "p25": 0.15142615379537977, "p50": 0.2763223977997799, "p75": 0.4517722253575359, "p95": 1.6299054529482935 }, "totalTokens": { "p05": 152419.26897689793, "p25": 250398.5033003298, "p50": 484363.37403740385, "p75": 929714.8008800887, "p95": 3672521.961386143 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "4d6aa9e9c8df931d54dba5956d94973c", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-medium", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (medium)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (medium)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.654867256637168, "inputTokens": 1089240.6017699116, "outputTokens": 21123.365781710916 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 17, "hardStopAttemptCount": 0, "recoveredAttemptCount": 17, "fallbackAttemptCount": 17, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.0456989247311828 }, "rewardHacking": null, "mean": { "reward": 0.448924731182796, "inputTokens": 567843.747311828, "outputTokens": 9186.077956989247 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 2, "hardStopAttemptCount": 0, "recoveredAttemptCount": 2, "fallbackAttemptCount": 2, "refusals": [ { "turn": 75, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 144, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.010101010101010102 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 54, "reviewedAttemptCount": 54, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 54 } ], "rate": 0 }, "mean": { "reward": 0.272727272727273, "inputTokens": 2728043.8636363638, "outputTokens": 40784.555555555555 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 19, "hardStopAttemptCount": 0, "recoveredAttemptCount": 19, "fallbackAttemptCount": 19, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 75, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 144, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.018599978277397632 }, "indexScore": 0.4588397535157457, "mean": { "reward": 0.4588397535157457, "costUsd": 0.6187507753025301, "agentWallTimeSec": 510.7721826182621, "steps": 22.20902090209021, "inputTokens": 1232830.7183718372, "outputTokens": 20520.774477447743, "cacheTokens": 1173805.1144114407, "cacheHitRate": 0.9341009883549257, "totalTokens": 1253351.4928492843 }, "sums": { "costUsd": 187.48148491666672 }, "percentiles": { "costUsd": { "p05": 0.1301400262926294, "p25": 0.2095640195269526, "p50": 0.41369848030803086, "p75": 0.7082445721672166, "p95": 2.060476164268427 }, "totalTokens": { "p05": 194819.09009900966, "p25": 320817.6897689771, "p50": 727342.763476347, "p75": 1444183.7959295923, "p95": 4695596.860066004 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "7a0dfd2a7a9d93056f860d4675586d66", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-6-sol", "display": { "agent": "Codex", "model": "GPT-6 Sol (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6 Sol (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 337, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.690265486725664, "inputTokens": 9754196.967551623, "outputTokens": 70300.33038348083 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.575268817204301, "inputTokens": 6133614.827956989, "outputTokens": 37933.720430107525 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 87, "reviewedAttemptCount": 87, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 87 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.434343434343434, "inputTokens": 16668544.737373738, "outputTokens": 93810.5303030303 } } ], "safety": { "attemptCount": 907, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5666259127577997, "mean": { "reward": 0.5666259127577997, "costUsd": 2.989729004180416, "agentWallTimeSec": 1338.0527051705185, "steps": 86.55005500550054, "inputTokens": 9778602.140814083, "outputTokens": 62175.62266226623, "cacheTokens": 9598928.887788778, "cacheHitRate": 0.9640119921972632, "totalTokens": 9840777.763476355 }, "sums": { "costUsd": 905.887888266666 }, "percentiles": { "costUsd": { "p05": 0.8303700961936205, "p25": 1.412352845434542, "p50": 2.0778670118811906, "p75": 3.503332819251925, "p95": 8.13729606064907 }, "totalTokens": { "p05": 1633543.1261826195, "p25": 3470166.2843784397, "p50": 5870785.018701871, "p75": 11366503.224422423, "p95": 31676596.306490637 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "4aee8aa2d4a0bad6068644f40edae235", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (high)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (high) ({'reasoning_effort': 'high'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.705014749262537, "inputTokens": 2382530.2064896757, "outputTokens": 31206.94395280236 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 14, "hardStopAttemptCount": 14, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 39, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03763440860215054 }, "rewardHacking": null, "mean": { "reward": 0.599462365591398, "inputTokens": 2421836.2786885244, "outputTokens": 17768.75956284153 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 1, "hardStopAttemptCount": 1, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.005050505050505051 } ], "rate": 0.005050505050505051 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 99, "reviewedAttemptCount": 99, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 99 } ], "rate": 0 }, "mean": { "reward": 0.5, "inputTokens": 4184679.611111111, "outputTokens": 45169.030303030304 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 15, "hardStopAttemptCount": 15, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 39, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.0016835016835016836 } ], "rate": 0.014228304550885196 }, "indexScore": 0.6014923716179784, "mean": { "reward": 0.6014923716179784, "costUsd": 0.8892413135295484, "agentWallTimeSec": 799.7652123212325, "steps": 38.97909790979098, "inputTokens": 2791163.2548648305, "outputTokens": 28748.735486663423, "cacheTokens": 2665415.3918375443, "cacheHitRate": 0.9278388366571141, "totalTokens": 2819911.9903514963 }, "sums": { "costUsd": 268.0356914333333 }, "percentiles": { "costUsd": { "p05": 0.312068081210121, "p25": 0.4989411210121013, "p50": 0.6891893533553354, "p75": 0.9873919497799788, "p95": 2.1023536615841585 }, "totalTokens": { "p05": 527228.1344334432, "p25": 1136802.8481848184, "p50": 1870636.3718371852, "p75": 3261089.490649065, "p95": 6454477.248184814 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "fa589dad83ffcdd6cce72a467e552063", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (low)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (low) ({'reasoning_effort': 'low'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.675516224188791, "inputTokens": 1135333.3510324485, "outputTokens": 12865.970501474927 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 9, "hardStopAttemptCount": 9, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 4, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.024193548387096774 }, "rewardHacking": null, "mean": { "reward": 0.551075268817204, "inputTokens": 1278601.31147541, "outputTokens": 9084.045081967213 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 98, "reviewedAttemptCount": 98, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 98 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.48989898989899, "inputTokens": 3454474.217171717, "outputTokens": 28368.58080808081 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 9, "hardStopAttemptCount": 9, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 4, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.008064516129032258 }, "indexScore": 0.5721634943016617, "mean": { "reward": 0.5721634943016617, "costUsd": 0.49880364510270725, "agentWallTimeSec": 517.8138943894386, "steps": 35.74037403740374, "inputTokens": 1699123.8601417518, "outputTokens": 14695.05805334632, "cacheTokens": 1627295.6717524226, "cacheHitRate": 0.9135762473536158, "totalTokens": 1713818.9181950986 }, "sums": { "costUsd": 150.4073521333333 }, "percentiles": { "costUsd": { "p05": 0.14515691546754672, "p25": 0.2084983163916393, "p50": 0.2965749825082511, "p75": 0.5361287875687569, "p95": 1.465534774675468 }, "totalTokens": { "p05": 240942.00825082537, "p25": 418718.0632563254, "p50": 753671.7865786579, "p75": 1840610.7161716148, "p95": 5662162.880087997 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "55b029d6971b4d378335bb168eb98b3b", "isDefault": true, "isHighlighted": false, "agentName": "Kimi Code CLI", "provider": "moonshotai", "hostModelSlug": "moonshot_kimi-k3", "display": { "agent": "Kimi Code CLI", "model": "Kimi K3", "creator": { "agent": "Moonshot AI", "model": "Moonshot AI" } }, "displayLabel": "Kimi Code CLI - Kimi K3", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.684365781710915, "inputTokens": 8571957.041297935, "outputTokens": 75080.34808259587 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.661290322580645, "inputTokens": 4719526.534946237, "outputTokens": 49977.29838709677 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 42, "reviewedAttemptCount": 42, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 42 } ], "rate": 0 }, "mean": { "reward": 0.212121212121212, "inputTokens": 14964824.787878787, "outputTokens": 205571.03535353535 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.519259105470924, "mean": { "reward": 0.519259105470924, "costUsd": 5.047195443300332, "agentWallTimeSec": 3689.3441947194724, "steps": 173.36028602860284, "inputTokens": 8387890.666666667, "outputTokens": 93230.86688668867, "cacheTokens": 7968496.133333323, "cacheHitRate": 0.9500000000000017, "totalTokens": 8481121.533553353 }, "sums": { "costUsd": 1529.30021932 }, "percentiles": { "costUsd": { "p05": 1.1201608924026403, "p25": 2.1660128471864684, "p50": 4.013249413745875, "p75": 6.4578525155775575, "p95": 11.098114784331683 }, "totalTokens": { "p05": 1514640.538393838, "p25": 3247568.457095711, "p50": 6455899.625962595, "p75": 11496187.961496139, "p95": 19359817.106380656 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.36.0", "dateReleased": "2026-08-13" }, "max": { "version": "0.36.1", "dateReleased": "2026-08-14" } }, "swe-atlas-qna": { "min": { "version": "0.26.0", "dateReleased": "2026-07-16" }, "max": { "version": "0.26.0", "dateReleased": "2026-07-16" } }, "terminal-bench-v4": { "min": { "version": "0.36.1", "dateReleased": "2026-08-14" }, "max": { "version": "0.36.1", "dateReleased": "2026-08-14" } } } }, { "id": "45493c2782f1bd94f6d30d4911f317a0", "isDefault": true, "isHighlighted": false, "agentName": "Opencode", "provider": "zai", "hostModelSlug": "zai_glm-5-3", "display": { "agent": "Opencode", "model": "GLM-5.3", "creator": { "agent": "Opencode", "model": "Z.ai" } }, "displayLabel": "Opencode - GLM-5.3 ({'reasoning_effort': 'max'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.613569321533923, "inputTokens": 16638607.92920354, "outputTokens": 75199.58702064896 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.594086021505376, "inputTokens": 6141052.228494624, "outputTokens": 28542.190860215054 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 80, "reviewedAttemptCount": 80, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 80 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.398989898989899, "inputTokens": 25175801.106060605, "outputTokens": 158541.98484848483 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5355484140097326, "mean": { "reward": 0.5355484140097326, "costUsd": 4.241385833751375, "agentWallTimeSec": 2887.9556523652373, "steps": 107.04950495049505, "inputTokens": 14202165.166116614, "outputTokens": 74259.26072607259, "cacheTokens": 14007356.268426869, "cacheHitRate": 0.9745705628087887, "totalTokens": 14276424.426842693 }, "sums": { "costUsd": 1285.139907626667 }, "percentiles": { "costUsd": { "p05": 0.889673751515953, "p25": 1.8921473049724982, "p50": 2.9852793977557757, "p75": 5.713520085478545, "p95": 10.917436082209012 }, "totalTokens": { "p05": 2410964.3325632564, "p25": 5510060.859735972, "p50": 9868718.469746958, "p75": 19057305.483498357, "p95": 38946948.10616063 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } } } }, { "id": "70a5d300021b9986f77d4198272ef1a5", "isDefault": false, "isHighlighted": false, "agentName": "Grok Build", "provider": "xai", "hostModelSlug": "xai_grok-4-6-xhigh", "display": { "agent": "Grok Build", "model": "Grok 4.6 (xhigh)", "creator": { "agent": "xAI", "model": "xAI" } }, "displayLabel": "Grok Build - Grok 4.6 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.648967551622419, "inputTokens": 6323126.3598820055, "outputTokens": 69583.08554572272 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 3, "hardStopAttemptCount": 3, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.008064516129032258 }, "rewardHacking": null, "mean": { "reward": 0.583333333333333, "inputTokens": 2992096.1370967743, "outputTokens": 26161.698924731183 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 36, "reviewedAttemptCount": 36, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 36 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.176767676767677, "inputTokens": 8745920.277777778, "outputTokens": 88596.601010101 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 3, "hardStopAttemptCount": 3, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.0026881720430107525 }, "indexScore": 0.4696895205744764, "mean": { "reward": 0.4696895205744764, "costUsd": 3.5701432805280535, "agentWallTimeSec": 1167.3387480748086, "steps": 112.06160616061608, "inputTokens": 5487669.762376238, "outputTokens": 55954.83498349835, "cacheTokens": 5160616.836083612, "cacheHitRate": 0.9057792630165156, "totalTokens": 5543624.597359739 }, "sums": { "costUsd": 1081.753414000001 }, "percentiles": { "costUsd": { "p05": 1.175140282068208, "p25": 2.007830724972496, "p50": 2.783706352035202, "p75": 4.600172847084706, "p95": 7.456153753135318 }, "totalTokens": { "p05": 1341131.6409240938, "p25": 2646561.652915294, "p50": 4024738.3938393835, "p75": 7158372.716171614, "p95": 12765170.414081411 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } }, "swe-atlas-qna": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } }, "terminal-bench-v4": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } } } }, { "id": "b3c56ec69185d544992fd578edb75e8c", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-5-6-luna", "display": { "agent": "Codex", "model": "GPT-5.6 Luna (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-5.6 Luna (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.663716814159292, "inputTokens": 16195750.386430679, "outputTokens": 75781.00294985251 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.486559139784946, "inputTokens": 7864729.142473118, "outputTokens": 31795.564516129034 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 29, "reviewedAttemptCount": 29, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 29 } ], "rate": 0 }, "mean": { "reward": 0.146464646464646, "inputTokens": 25363172.36868687, "outputTokens": 137870.4292929293 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4322468668029613, "mean": { "reward": 0.4322468668029613, "costUsd": 0.43802256393839384, "agentWallTimeSec": 1412.260883388339, "steps": 121.51375137513753, "inputTokens": 14783219.74807481, "outputTokens": 71304.79097909792, "cacheTokens": 14535765.897689763, "cacheHitRate": 0.9718803286808345, "totalTokens": 14854524.539053895 }, "sums": { "costUsd": 132.7208368733333 }, "percentiles": { "costUsd": { "p05": 0.11470550527832782, "p25": 0.19050873430693036, "p50": 0.2971945570627061, "p75": 0.5478214921672169, "p95": 1.2523768718877888 }, "totalTokens": { "p05": 2583111.566226621, "p25": 5468368.5495049525, "p50": 9575964.7370737, "p75": 19574390.483498316, "p95": 48106178.56754682 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "542590ecc1aa12a718d158565d464daa", "isDefault": true, "isHighlighted": false, "agentName": "Codex", "provider": "deepseek", "hostModelSlug": "deepseek_deepseek-v4-pro-0813", "display": { "agent": "Codex", "model": "DeepSeek V4 Pro 0813 (max)", "creator": { "agent": "OpenAI", "model": "DeepSeek" } }, "displayLabel": "Codex - DeepSeek V4 Pro 0813 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.572271386430678, "inputTokens": 21857745.433628317, "outputTokens": 116676.62241887905 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.618279569892473, "inputTokens": 9901327.01075269, "outputTokens": 46364.06182795699 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 21, "reviewedAttemptCount": 21, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 21 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.101010101010101, "inputTokens": 54448314.87373737, "outputTokens": 247887.35606060605 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4305203524444174, "mean": { "reward": 0.4305203524444174, "costUsd": 0.23804610720792058, "agentWallTimeSec": 2417.5500099009905, "steps": 138.14631463146316, "inputTokens": 24063625.627062704, "outputTokens": 116482.400990099, "cacheTokens": 23948932.43564355, "cacheHitRate": 0.9898485564269163, "totalTokens": 24180108.028052814 }, "sums": { "costUsd": 72.127970484 }, "percentiles": { "costUsd": { "p05": 0.08553589994158414, "p25": 0.13481908242684273, "p50": 0.20057701675687584, "p75": 0.30849491996424616, "p95": 0.4808581146649064 }, "totalTokens": { "p05": 3736359.9761276133, "p25": 8302592.635863575, "p50": 17833020.419141915, "p75": 32302640.28272825, "p95": 63649800.394829504 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.147.0", "dateReleased": "2026-08-07" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } } ], "aa_intelligence_index": [ { "name": "GLM-4.5V (Non-reasoning)", "shortName": "GLM-4.5V (non-reasoning)", "slug": "glm-4-5v", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 6.69669796944385, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0347544022242817, "critpt": 0, "lcr": 0, "gpqa": 0.572727272727273, "omniscience": -55.583333333333336, "omniscienceAccuracy": 0.18216666666666667, "omniscienceNonHallucination": 0.09761565111065829, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 1.8, "cacheHitPrice": 0.11, "cacheWritePrice": null, "contextWindowTokens": 64000, "medianOutputTokensPerSecond": 47.3466863308784, "medianTimeToFirstTokenSeconds": 2.82427125950002, "isOpenWeights": true }, { "name": "Gemini 3.5 Flash (High)", "shortName": "Gemini 3.5 Flash (high)", "slug": "gemini-3-5-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 32.5983283695905, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.5625413241435933, "terminalBench40": 0.0656565656565657, "terminalBench21": 0.786516853932584, "terminalbenchHard": 0.409090909090909, "scicode": 0.539351851851852, "hle": 0.42678405931418, "critpt": 0.131428571428571, "lcr": 0.733333333333333, "gpqa": 0.922222222222222, "omniscience": 21.183333333333334, "omniscienceAccuracy": 0.514, "omniscienceNonHallucination": 0.37825788751714673, "gdpvalNormalized": 0.3424, "price1mInputTokens": 1.5, "price1mOutputTokens": 9, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 199.762602436871, "medianTimeToFirstTokenSeconds": 16.5143645735, "isOpenWeights": false }, { "name": "MiMo-V2.5-Pro", "shortName": "MiMo-V2.5-Pro", "slug": "mimo-v2-5-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 25.9868888896511, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.054072740415336835, "terminalBench40": 0, "terminalBench21": 0.651685393258427, "terminalbenchHard": 0.431818181818182, "scicode": 0.505787037037037, "hle": 0.356811862835959, "critpt": 0.04, "lcr": 0.796666666666667, "gpqa": 0.865656565656566, "omniscience": 3.25, "omniscienceAccuracy": 0.22416666666666665, "omniscienceNonHallucination": 0.7529538131041891, "gdpvalNormalized": 0.30351, "price1mInputTokens": 0.435, "price1mOutputTokens": 0.87, "cacheHitPrice": 0.0036, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 30.7171789041459, "medianTimeToFirstTokenSeconds": 2.469515585, "isOpenWeights": true }, { "name": "JT-35B-Flash", "shortName": "JT-35B-Flash", "slug": "jt-35b-flash", "isReasoning": false, "deprecated": false, "modelCreatorName": "China Mobile", "intelligenceIndex": 18.6626376411117, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.287878787878788, "scicode": null, "hle": 0.0644114921223355, "critpt": 0, "lcr": 0.656666666666667, "gpqa": 0.829292929292929, "omniscience": -24.216666666666665, "omniscienceAccuracy": 0.24183333333333334, "omniscienceNonHallucination": 0.3616179380083535, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "GPT-6 Sol (Low)", "shortName": "GPT-6 Sol (low)", "slug": "gpt-6-sol-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 33.9008505585733, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.1326186130291875, "terminalBench40": 0.0909090909090909, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.502314814814815, "hle": 0.349397590361446, "critpt": 0.162857142857143, "lcr": 0.793333333333333, "gpqa": null, "omniscience": 26.516666666666666, "omniscienceAccuracy": 0.5125, "omniscienceNonHallucination": 0.49264957264957265, "gdpvalNormalized": 0.33779499999999996, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 872000, "medianOutputTokensPerSecond": 56.345849780237, "medianTimeToFirstTokenSeconds": 2.13548850649977, "isOpenWeights": false }, { "name": "Mercury 2", "shortName": "Mercury 2", "slug": "mercury-2", "isReasoning": true, "deprecated": true, "modelCreatorName": "Inception", "intelligenceIndex": 13.7691315513812, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.273408239700375, "terminalbenchHard": 0.265151515151515, "scicode": 0.377314814814815, "hle": 0.171455050973123, "critpt": 0.00848979591836735, "lcr": 0.436666666666667, "gpqa": 0.76969696969697, "omniscience": -50.71666666666667, "omniscienceAccuracy": 0.21183333333333335, "omniscienceNonHallucination": 0.08775639670120527, "gdpvalNormalized": 0, "price1mInputTokens": 0.25, "price1mOutputTokens": 0.75, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 845.733777993096, "medianTimeToFirstTokenSeconds": 4.53513061900003, "isOpenWeights": false }, { "name": "Qwen3.5 Omni Plus", "shortName": "Qwen3.5 Omni Plus", "slug": "qwen3-5-omni-plus", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 20.3839890713777, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.212121212121212, "scicode": null, "hle": 0.148748841519926, "critpt": 0.00563265306122449, "lcr": 0.636666666666667, "gpqa": 0.826262626262626, "omniscience": -13.116666666666667, "omniscienceAccuracy": 0.17433333333333334, "omniscienceNonHallucination": 0.6299959628582963, "gdpvalNormalized": null, "price1mInputTokens": 0.4, "price1mOutputTokens": 4.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 79.9302933971399, "medianTimeToFirstTokenSeconds": 2.25335172649994, "isOpenWeights": false }, { "name": "Nova 2.0 Omni (Medium)", "shortName": "Nova 2.0 Omni (medium)", "slug": "nova-2-0-omni-reasoning-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 13.6465213725579, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0695088044485635, "critpt": 0, "lcr": 0.596666666666667, "gpqa": 0.759595959595959, "omniscience": -59.083333333333336, "omniscienceAccuracy": 0.177, "omniscienceNonHallucination": 0.06703118671526931, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "GPT-5.2 Codex (Xhigh)", "shortName": "GPT-5.2 Codex (xhigh)", "slug": "gpt-5-2-codex", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 28.5016971971953, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.371212121212121, "scicode": null, "hle": 0.357275254865616, "critpt": 0.0866938775510204, "lcr": 0.823333333333333, "gpqa": 0.898989898989899, "omniscience": -2.1666666666666665, "omniscienceAccuracy": 0.4106666666666667, "omniscienceNonHallucination": 0.26640271493212675, "gdpvalNormalized": null, "price1mInputTokens": 1.75, "price1mOutputTokens": 14, "cacheHitPrice": 0.175, "cacheWritePrice": null, "contextWindowTokens": 400000, "isOpenWeights": false }, { "name": "Qwen3.5 122B A10B (Non-reasoning)", "shortName": "Qwen3.5 122B A10B (non-reasoning)", "slug": "qwen3-5-122b-a10b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 17.7480154665397, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.471910112359551, "terminalbenchHard": 0.295454545454545, "scicode": null, "hle": 0.159406858202039, "critpt": 0.00857142857142857, "lcr": 0.613333333333333, "gpqa": 0.827272727272727, "omniscience": -54.96666666666667, "omniscienceAccuracy": 0.19116666666666668, "omniscienceNonHallucination": 0.0840717082217185, "gdpvalNormalized": 0.09738, "price1mInputTokens": 0.4, "price1mOutputTokens": 3.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 138.439065569427, "medianTimeToFirstTokenSeconds": 2.34552344250003, "isOpenWeights": true }, { "name": "Gemma 4 12B (Non-reasoning)", "shortName": "Gemma 4 12B (non-reasoning)", "slug": "gemma-4-12b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 9.37989472421056, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.113636363636364, "scicode": null, "hle": 0.0634847080630213, "critpt": 0, "lcr": 0.35, "gpqa": 0.660606060606061, "omniscience": -52.8, "omniscienceAccuracy": 0.11883333333333333, "omniscienceNonHallucination": 0.26593531303196516, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 116.511826203535, "medianTimeToFirstTokenSeconds": 2.47742370950002, "isOpenWeights": true }, { "name": "Gemini 3.5 Flash (Minimal)", "shortName": "Gemini 3.5 Flash (minimal)", "slug": "gemini-3-5-flash-minimal", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 23.848790860577, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.462121212121212, "scicode": null, "hle": 0.240963855421687, "critpt": 0.0142857142857143, "lcr": 0.613333333333333, "gpqa": 0.828282828282828, "omniscience": 0.6666666666666666, "omniscienceAccuracy": 0.43016666666666664, "omniscienceNonHallucination": 0.2568002339865457, "gdpvalNormalized": null, "price1mInputTokens": 1.5, "price1mOutputTokens": 9, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 196.843216264737, "medianTimeToFirstTokenSeconds": 0.964426676999999, "isOpenWeights": false }, { "name": "Claude 3.5 Haiku", "shortName": "Claude 3.5 Haiku", "slug": "claude-3-5-haiku", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 8.938372415746, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.101123595505618, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0363, "critpt": 0, "lcr": 0.273333333333333, "gpqa": 0.408080808080808, "omniscience": -22.516666666666666, "omniscienceAccuracy": 0.132, "omniscienceNonHallucination": 0.5885176651305684, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "GPT-3.5 Turbo", "shortName": "GPT-3.5 Turbo", "slug": "gpt-35-turbo", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 5.48558234391291, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.296969696969697, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.5, "price1mOutputTokens": 1.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": false }, { "name": "Gemini 2.0 Flash Thinking Experimental (Jan '25)", "shortName": "Gemini 2.0 Flash Thinking exp. (Jan)", "slug": "gemini-2-0-flash-thinking-exp-0121", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 9.41808559069261, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0635, "critpt": null, "lcr": null, "gpqa": 0.701010101010101, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Claude Sonnet 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)", "shortName": "Claude Sonnet 5.5 (medium with fallback)", "slug": "claude-sonnet-5-5-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 40.7386909594307, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5861822401191022, "terminalBench40": 0.297979797979798, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.528935185185185, "hle": 0.39805375347544, "critpt": 0.168571428571429, "lcr": 0.763333333333333, "gpqa": null, "omniscience": 20.1, "omniscienceAccuracy": 0.4716666666666667, "omniscienceNonHallucination": 0.48769716088328074, "gdpvalNormalized": 0.3961, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 91.6939430117019, "medianTimeToFirstTokenSeconds": 1.34296612500003, "isOpenWeights": false }, { "name": "Qwen2.5 Coder Instruct 32B", "shortName": "Qwen2.5 Coder 32B", "slug": "qwen2-5-coder-32b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.74052379746597, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0351, "critpt": null, "lcr": null, "gpqa": 0.417171717171717, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Sonar Reasoning", "shortName": "Sonar Reasoning", "slug": "sonar-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Perplexity", "intelligenceIndex": 8.6724759846559, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.623232323232323, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 127000, "isOpenWeights": false }, { "name": "Gemma 3 270M", "shortName": "Gemma 3 270M", "slug": "gemma-3-270m", "isReasoning": false, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 5.11705960692171, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.036144578313253, "critpt": 0, "lcr": 0, "gpqa": 0.224242424242424, "omniscience": -29.266666666666666, "omniscienceAccuracy": 0.008833333333333334, "omniscienceNonHallucination": 0.6958130149655288, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Grok 4.20 0309 (Reasoning)", "shortName": "Grok 4.20 0309", "slug": "grok-4-20-0309", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 25.2162736582441, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.409090909090909, "scicode": null, "hle": 0.323911028730306, "critpt": 0.06, "lcr": 0.676666666666667, "gpqa": 0.884848484848485, "omniscience": 12.95, "omniscienceAccuracy": 0.28883333333333333, "omniscienceNonHallucination": 0.7759550035153504, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "GPT-5.1 Codex (High)", "shortName": "GPT-5.1 Codex (high)", "slug": "gpt-5-1-codex", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 23.6972790075055, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.348484848484849, "scicode": null, "hle": 0.257182576459685, "critpt": 0.0571428571428571, "lcr": 0.693333333333333, "gpqa": 0.85959595959596, "omniscience": -6.5, "omniscienceAccuracy": 0.399, "omniscienceNonHallucination": 0.2279534109816972, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "isOpenWeights": false }, { "name": "Muse Spark 1.2 (Xhigh)", "shortName": "Muse Spark 1.2 (xhigh)", "slug": "muse-spark-1-2", "isReasoning": true, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 39.5758607970943, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.9746948879931989, "terminalBench40": 0.0707070707070707, "terminalBench21": 0.801498127340824, "terminalbenchHard": null, "scicode": 0.574074074074074, "hle": 0.454587581093605, "critpt": 0.177142857142857, "lcr": 0.79, "gpqa": 0.904040404040404, "omniscience": 27.2, "omniscienceAccuracy": 0.4538333333333333, "omniscienceNonHallucination": 0.6670735428745804, "gdpvalNormalized": 0.49080999999999997, "price1mInputTokens": 1.25, "price1mOutputTokens": 4.25, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1048576, "medianOutputTokensPerSecond": 231.203326063637, "medianTimeToFirstTokenSeconds": 12.1356657739998, "isOpenWeights": false }, { "name": "GPT-5.6 Luna (Low)", "shortName": "GPT-5.6 Luna (low)", "slug": "gpt-5-6-luna-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 21.0124783424265, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.009827249754655997, "terminalBench40": 0, "terminalBench21": 0.434456928838951, "terminalbenchHard": null, "scicode": 0.460648148148148, "hle": 0.198331788693234, "critpt": 0.0257142857142857, "lcr": 0.7, "gpqa": 0.835353535353535, "omniscience": -14.65, "omniscienceAccuracy": 0.39616666666666667, "omniscienceNonHallucination": 0.10129726745790779, "gdpvalNormalized": 0.240485, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.02, "cacheWritePrice": 0.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 105.584274245164, "medianTimeToFirstTokenSeconds": 1.59970152799996, "isOpenWeights": false }, { "name": "Claude Fable 5.1 (Adaptive Reasoning, Low Effort, Default Fallback)", "shortName": "Claude Fable 5.1 (low with fallback)", "slug": "claude-fable-5-1-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 46.8163422014862, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.371025320183763, "terminalBench40": 0.404040404040404, "terminalBench21": 0.850187265917603, "terminalbenchHard": null, "scicode": 0.56712962962963, "hle": 0.48887859128823, "critpt": 0.277142857142857, "lcr": 0.823333333333333, "gpqa": 0.880808080808081, "omniscience": 34.13333333333333, "omniscienceAccuracy": 0.6023333333333334, "omniscienceNonHallucination": 0.34367141659681477, "gdpvalNormalized": 0.47478499999999996, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 0.25, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 47.3248877802417, "medianTimeToFirstTokenSeconds": 4.54629413199985, "isOpenWeights": false }, { "name": "Mistral Medium 3.1", "shortName": "Mistral Medium 3.1", "slug": "mistral-medium-3-1", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 9.18437902519296, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.138576779026217, "terminalbenchHard": 0.106060606060606, "scicode": 0.324074074074074, "hle": 0.0468025949953661, "critpt": 0, "lcr": 0.426666666666667, "gpqa": 0.587878787878788, "omniscience": -46.28333333333333, "omniscienceAccuracy": 0.2075, "omniscienceNonHallucination": 0.15415352260778126, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "GPT-5 nano (Minimal)", "shortName": "GPT-5 nano (minimal)", "slug": "gpt-5-nano-minimal", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 7.08879356535892, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0398517145505097, "critpt": 0, "lcr": 0.2, "gpqa": 0.428282828282828, "omniscience": -64.06666666666666, "omniscienceAccuracy": 0.13083333333333333, "omniscienceNonHallucination": 0.11236816874400768, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.005, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 144.153950559737, "medianTimeToFirstTokenSeconds": 0.888107631999993, "isOpenWeights": false }, { "name": "Gemini 2.5 Pro Preview (May' 25)", "shortName": "Gemini 2.5 Pro (May)", "slug": "gemini-2-5-pro-05-06", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 14.5215571266208, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.1993, "critpt": null, "lcr": null, "gpqa": 0.822222222222222, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "o3-mini (High)", "shortName": "o3-mini (high)", "slug": "o3-mini-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 10.9643866525084, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.0449438202247191, "terminalbenchHard": 0.0606060606060606, "scicode": 0.428240740740741, "hle": 0.1204, "critpt": 0.00285714285714286, "lcr": 0.43, "gpqa": 0.772727272727273, "omniscience": -42.56666666666667, "omniscienceAccuracy": 0.21266666666666667, "omniscienceNonHallucination": 0.1892464013547841, "gdpvalNormalized": 0, "price1mInputTokens": 1.1, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.55, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 201.741187495778, "medianTimeToFirstTokenSeconds": 20.5693831985, "isOpenWeights": false }, { "name": "o1-preview", "shortName": "o1-preview", "slug": "o1-preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 11.3759356934695, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.764646464646465, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 16.5, "price1mOutputTokens": 66, "cacheHitPrice": 8.25, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "DeepSeek V3.2 Exp (Non-reasoning)", "shortName": "DeepSeek V3.2 Exp (non-reasoning)", "slug": "deepseek-v3-2-0925", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 13.8833794351459, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.25, "scicode": null, "hle": 0.0903614457831325, "critpt": 0.0142857142857143, "lcr": 0.446666666666667, "gpqa": 0.738383838383838, "omniscience": -48.2, "omniscienceAccuracy": 0.22833333333333333, "omniscienceNonHallucination": 0.07948164146868253, "gdpvalNormalized": null, "price1mInputTokens": 0.28, "price1mOutputTokens": 0.42, "cacheHitPrice": 0.028, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Apodex 1.1", "shortName": "Apodex 1.1", "slug": "apodex-1-1", "isReasoning": true, "deprecated": false, "modelCreatorName": "Apodex", "intelligenceIndex": 26.4103686249117, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.4637245706928438, "terminalBench40": 0, "terminalBench21": 0.696629213483146, "terminalbenchHard": null, "scicode": 0.454861111111111, "hle": 0.340593141797961, "critpt": 0.0457142857142857, "lcr": 0.793333333333333, "gpqa": 0.863636363636364, "omniscience": -21.9, "omniscienceAccuracy": 0.31683333333333336, "omniscienceNonHallucination": 0.2156623566723591, "gdpvalNormalized": 0.34429, "price1mInputTokens": 0.3, "price1mOutputTokens": 3, "cacheHitPrice": 0.03, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "GPT-5.6 Sol (Low)", "shortName": "GPT-5.6 Sol (low)", "slug": "gpt-5-6-sol-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 33.4731387646002, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.2606419442090313, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.767790262172285, "terminalbenchHard": 0.606060606060606, "scicode": 0.563657407407407, "hle": 0.393883225208526, "critpt": 0.148571428571429, "lcr": 0.78, "gpqa": 0.897979797979798, "omniscience": 18.866666666666667, "omniscienceAccuracy": 0.5716666666666667, "omniscienceNonHallucination": 0.10583657587548634, "gdpvalNormalized": 0.39467499999999994, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.4, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 70.8003551397125, "medianTimeToFirstTokenSeconds": 2.38993325450003, "isOpenWeights": false }, { "name": "GPT-6.1 Sol (Xhigh)", "shortName": "GPT-6.1 Sol (xhigh)", "slug": "gpt-6-1-sol-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 51.0377679093761, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.39286117158850653, "terminalBench40": 0.54040404040404, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.556712962962963, "hle": 0.526413345690454, "critpt": 0.317142857142857, "lcr": 0.796666666666667, "gpqa": null, "omniscience": 40.86666666666667, "omniscienceAccuracy": 0.6081666666666666, "omniscienceNonHallucination": 0.49085495533815393, "gdpvalNormalized": 0.5052000000000001, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.1, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 63.2958464383106, "medianTimeToFirstTokenSeconds": 107.764038148, "isOpenWeights": false }, { "name": "Qwen3 Coder 480B A35B Instruct", "shortName": "Qwen3 Coder 480B", "slug": "qwen3-coder-480b-a35b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 11.8990256096549, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.189393939393939, "scicode": null, "hle": 0.0454124189063948, "critpt": 0, "lcr": 0.456666666666667, "gpqa": 0.618181818181818, "omniscience": -22.2, "omniscienceAccuracy": 0.15683333333333332, "omniscienceNonHallucination": 0.5507017197074521, "gdpvalNormalized": null, "price1mInputTokens": 1.5, "price1mOutputTokens": 7.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 56.1944038179381, "medianTimeToFirstTokenSeconds": 3.16295465099998, "isOpenWeights": true }, { "name": "Gemini 3.5 Flash-Lite", "shortName": "Gemini 3.5 Flash-Lite", "slug": "gemini-3-5-flash-lite", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 22.1685424839812, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.12352689651593446, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.535580524344569, "terminalbenchHard": null, "scicode": 0.413194444444444, "hle": 0.188137164040778, "critpt": 0, "lcr": 0.76, "gpqa": 0.838383838383838, "omniscience": 5.233333333333333, "omniscienceAccuracy": 0.2946666666666667, "omniscienceNonHallucination": 0.6564272211720227, "gdpvalNormalized": 0.23476, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.03, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 338.855574239312, "medianTimeToFirstTokenSeconds": 9.31839539349999, "isOpenWeights": false }, { "name": "Step 5 Preview", "shortName": "Step 5 Preview", "slug": "step-5", "isReasoning": true, "deprecated": false, "modelCreatorName": "StepFun", "intelligenceIndex": 43.7343049141614, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.7174901509283937, "terminalBench40": 0.333333333333333, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.58912037037037, "hle": 0.464782205746061, "critpt": 0.208571428571429, "lcr": 0.883333333333333, "gpqa": null, "omniscience": 16.383333333333333, "omniscienceAccuracy": 0.41533333333333333, "omniscienceNonHallucination": 0.5698403648802737, "gdpvalNormalized": 0.53306, "price1mInputTokens": 1, "price1mOutputTokens": 2.7, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 85.7941277174153, "medianTimeToFirstTokenSeconds": 3.12428767099991, "isOpenWeights": false }, { "name": "Qwen3.5 2B (Non-reasoning)", "shortName": "Qwen3.5 2B (non-reasoning)", "slug": "qwen3-5-2b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.18559190112645, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0495829471733086, "critpt": 0, "lcr": 0.14, "gpqa": 0.438383838383838, "omniscience": -83.13333333333334, "omniscienceAccuracy": 0.07383333333333333, "omniscienceNonHallucination": 0.02267410473276943, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Devstral 2", "shortName": "Devstral 2", "slug": "devstral-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 8.61676079574885, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.303370786516854, "terminalbenchHard": 0.189393939393939, "scicode": 0.327546296296296, "hle": 0.0356811862835959, "critpt": 0, "lcr": 0.323333333333333, "gpqa": 0.593939393939394, "omniscience": -46.7, "omniscienceAccuracy": 0.20816666666666667, "omniscienceNonHallucination": 0.1473374026520733, "gdpvalNormalized": 0.01569, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Claude 3.5 Sonnet (Oct '24)", "shortName": "Claude 3.5 Sonnet (Oct)", "slug": "claude-35-sonnet", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 7.87908981990033, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0369, "critpt": null, "lcr": null, "gpqa": 0.598989898989899, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "DeepSeek V3.1 Terminus (Reasoning)", "shortName": "DeepSeek V3.1 Terminus", "slug": "deepseek-v3-1-terminus-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 14.7640236054576, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.449438202247191, "terminalbenchHard": 0.303030303030303, "scicode": 0.37962962962963, "hle": 0.163577386468953, "critpt": 0.0171428571428571, "lcr": 0.693333333333333, "gpqa": 0.791919191919192, "omniscience": -26.4, "omniscienceAccuracy": 0.277, "omniscienceNonHallucination": 0.2517289073305671, "gdpvalNormalized": 0.09908999999999997, "price1mInputTokens": 1.635, "price1mOutputTokens": 2.75, "cacheHitPrice": 1.5675, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3 Omni 30B A3B Instruct", "shortName": "Qwen3 Omni 30B A3B", "slug": "qwen3-omni-30b-a3b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.00611816793457, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.046339202965709, "critpt": 0, "lcr": 0, "gpqa": 0.62020202020202, "omniscience": -69.26666666666667, "omniscienceAccuracy": 0.14333333333333334, "omniscienceNonHallucination": 0.024124513618677068, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 0.97, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "medianOutputTokensPerSecond": 95.7046692510887, "medianTimeToFirstTokenSeconds": 1.89523132400001, "isOpenWeights": true }, { "name": "Grok Build 0.1 0616", "shortName": "Grok Build 0.1 0616", "slug": "grok-build-0-1-06-16", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 27.1714438189407, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.52059925093633, "terminalbenchHard": null, "scicode": null, "hle": 0.382761816496756, "critpt": 0.0914285714285714, "lcr": 0.746666666666667, "gpqa": 0.894949494949495, "omniscience": 6.366666666666666, "omniscienceAccuracy": 0.5151666666666667, "omniscienceNonHallucination": 0.06875214850464073, "gdpvalNormalized": 0.27595500000000006, "price1mInputTokens": 1, "price1mOutputTokens": 2, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 48.0449418042921, "medianTimeToFirstTokenSeconds": 0.529561604500024, "isOpenWeights": false }, { "name": "Granite 4.2 3B", "shortName": "Granite 4.2 3B", "slug": "granite-4-2-3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 9.05588232376756, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.005980247671024821, "terminalBench40": 0, "terminalBench21": 0.138576779026217, "terminalbenchHard": null, "scicode": 0.253472222222222, "hle": 0.0662650602409639, "critpt": 0, "lcr": 0.243333333333333, "gpqa": 0.558585858585859, "omniscience": -14.716666666666667, "omniscienceAccuracy": 0.092, "omniscienceNonHallucination": 0.7366005873715125, "gdpvalNormalized": 0, "price1mInputTokens": 0.03, "price1mOutputTokens": 0.12, "cacheHitPrice": 0.0075, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 223.407679492408, "medianTimeToFirstTokenSeconds": 0.519832298999972, "isOpenWeights": true }, { "name": "Nova 2.0 Pro Preview (Medium)", "shortName": "Nova 2.0 Pro Preview (medium)", "slug": "nova-2-0-pro-reasoning-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 14.1554948903666, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.295880149812734, "terminalbenchHard": 0.242424242424242, "scicode": null, "hle": 0.0940685820203892, "critpt": 0, "lcr": 0.64, "gpqa": 0.784848484848485, "omniscience": -49.05, "omniscienceAccuracy": 0.21966666666666668, "omniscienceNonHallucination": 0.08991883810337464, "gdpvalNormalized": 0, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.312, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 113.425614049736, "medianTimeToFirstTokenSeconds": 13.2361074145, "isOpenWeights": false }, { "name": "Inkling (Xhigh)", "shortName": "Inkling (xhigh)", "slug": "inkling", "isReasoning": true, "deprecated": false, "modelCreatorName": "Thinking Machines", "intelligenceIndex": 24.9847810999384, "intelligenceIndexIsEstimated": false, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.550561797752809, "terminalbenchHard": null, "scicode": 0.469907407407407, "hle": 0.318813716404078, "critpt": 0.0542857142857143, "lcr": 0.773333333333333, "gpqa": 0.871717171717172, "omniscience": 2, "omniscienceAccuracy": 0.4155, "omniscienceNonHallucination": 0.32335329341317365, "gdpvalNormalized": 0.28183500000000006, "price1mInputTokens": 1, "price1mOutputTokens": 4.05, "cacheHitPrice": 0.17, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 159.961223989711, "medianTimeToFirstTokenSeconds": 1.885070799, "isOpenWeights": true }, { "name": "Kimi K2.6", "shortName": "Kimi K2.6", "slug": "kimi-k2-6", "isReasoning": true, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 26.9791999742844, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.799769615295709, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.659176029962547, "terminalbenchHard": 0.439393939393939, "scicode": 0.515046296296296, "hle": 0.374884151992586, "critpt": 0.08, "lcr": 0.81, "gpqa": 0.911111111111111, "omniscience": 5.3, "omniscienceAccuracy": 0.326, "omniscienceNonHallucination": 0.5949554896142433, "gdpvalNormalized": 0.263045, "price1mInputTokens": 0.95, "price1mOutputTokens": 4, "cacheHitPrice": 0.16, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 68.8373613644831, "medianTimeToFirstTokenSeconds": 2.97406106899995, "isOpenWeights": true }, { "name": "DeepSeek R1 (Jan '25)", "shortName": "DeepSeek R1 (Jan)", "slug": "deepseek-r1-0120", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 11.4109820044475, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.2617212405141733, "terminalBench40": 0, "terminalBench21": 0.191011235955056, "terminalbenchHard": 0.0606060606060606, "scicode": 0.383101851851852, "hle": 0.085, "critpt": 0.00571428571428571, "lcr": 0.576666666666667, "gpqa": 0.708080808080808, "omniscience": -32.21666666666667, "omniscienceAccuracy": 0.30516666666666664, "omniscienceNonHallucination": 0.0971455984648597, "gdpvalNormalized": 0, "price1mInputTokens": 2, "price1mOutputTokens": 4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "DeepSeek R1 Distill Llama 8B", "shortName": "DeepSeek R1 Distill Llama 8B", "slug": "deepseek-r1-distill-llama-8b", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 6.4762216011156, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": 0, "gpqa": 0.302020202020202, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3 0.6B (Non-reasoning)", "shortName": "Qwen3 0.6B (non-reasoning)", "slug": "qwen3-0.6b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0493, "critpt": 0, "lcr": 0, "gpqa": 0.231313131313131, "omniscience": -84.95, "omniscienceAccuracy": 0.04583333333333333, "omniscienceNonHallucination": 0.06165938864628817, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Qwen3.5 397B A17B (Reasoning)", "shortName": "Qwen3.5 397B A17B", "slug": "qwen3-5-397b-a17b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 18.4204249294815, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.47457695829894075, "terminalBench40": 0, "terminalBench21": 0.51310861423221, "terminalbenchHard": 0.409090909090909, "scicode": 0.447916666666667, "hle": 0.289620018535681, "critpt": 0.0170612244897959, "lcr": 0.773333333333333, "gpqa": 0.892929292929293, "omniscience": -30.75, "omniscienceAccuracy": 0.308, "omniscienceNonHallucination": 0.11054913294797686, "gdpvalNormalized": 0.13977999999999996, "price1mInputTokens": 0.6, "price1mOutputTokens": 3.6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 88.8245858281296, "medianTimeToFirstTokenSeconds": 2.26559528650001, "isOpenWeights": true }, { "name": "LFM2 1.2B", "shortName": "LFM2 1.2B", "slug": "lfm2-1-2b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Liquid AI", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0556070435588508, "critpt": 0, "lcr": 0, "gpqa": 0.228282828282828, "omniscience": -72.43333333333334, "omniscienceAccuracy": 0.04933333333333333, "omniscienceNonHallucination": 0.18618513323983166, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Kimi K2.6 (Non-reasoning)", "shortName": "Kimi K2.6 (non-reasoning)", "slug": "kimi-k2-6-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 23.5695903898533, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.378787878787879, "scicode": null, "hle": 0.195551436515292, "critpt": 0.0142857142857143, "lcr": 0.696666666666667, "gpqa": 0.787878787878788, "omniscience": -9.183333333333334, "omniscienceAccuracy": 0.243, "omniscienceNonHallucination": 0.5576838397181858, "gdpvalNormalized": null, "price1mInputTokens": 0.95, "price1mOutputTokens": 4, "cacheHitPrice": 0.16, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 49.3193681135286, "medianTimeToFirstTokenSeconds": 2.75118541249998, "isOpenWeights": true }, { "name": "Magistral Small 1", "shortName": "Magistral Small 1", "slug": "magistral-small", "isReasoning": true, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 8.22425322380767, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0755329008341057, "critpt": 0, "lcr": 0, "gpqa": 0.641414141414141, "omniscience": -44.4, "omniscienceAccuracy": 0.18016666666666667, "omniscienceNonHallucination": 0.2386663956088636, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 40000, "isOpenWeights": true }, { "name": "Granite 4.2 30B", "shortName": "Granite 4.2 30B", "slug": "granite-4-2-30b", "isReasoning": true, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 14.8312405050789, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.265917602996255, "terminalbenchHard": null, "scicode": 0.378472222222222, "hle": 0.111677479147359, "critpt": 0.00285714285714286, "lcr": 0.49, "gpqa": 0.644444444444444, "omniscience": -12.916666666666666, "omniscienceAccuracy": 0.10083333333333333, "omniscienceNonHallucination": 0.7442075996292864, "gdpvalNormalized": 0.03187, "price1mInputTokens": 0.16, "price1mOutputTokens": 0.65, "cacheHitPrice": 0.04, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 77.1746930635512, "medianTimeToFirstTokenSeconds": 0.875138198999935, "isOpenWeights": true }, { "name": "K2 Horizon 7B", "shortName": "K2 Horizon 7B", "slug": "k2-horizon-7b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 20.5958819421354, "intelligenceIndexIsEstimated": false, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.441947565543071, "terminalbenchHard": null, "scicode": 0.275462962962963, "hle": 0.182113067655236, "critpt": 0.00285714285714286, "lcr": 0.696666666666667, "gpqa": 0.752525252525252, "omniscience": -16.566666666666666, "omniscienceAccuracy": 0.12766666666666668, "omniscienceNonHallucination": 0.6637371035536874, "gdpvalNormalized": 0.2557, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 524288, "isOpenWeights": true }, { "name": "MiMo-V2.6-Flash", "shortName": "MiMo-V2.6-Flash", "slug": "mimo-v2-6-flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "Xiaomi", "intelligenceIndex": 37.8843590141754, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.0621899731897297, "terminalBench40": 0.227272727272727, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.512731481481482, "hle": 0.350787766450417, "critpt": 0.12, "lcr": 0.743333333333333, "gpqa": null, "omniscience": -12.683333333333334, "omniscienceAccuracy": 0.27, "omniscienceNonHallucination": 0.4563926940639269, "gdpvalNormalized": 0.55003, "price1mInputTokens": 0.14, "price1mOutputTokens": 0.28, "cacheHitPrice": 0.0028, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 53.8110509480365, "medianTimeToFirstTokenSeconds": 3.90881981050004, "isOpenWeights": true }, { "name": "LFM 40B", "shortName": "LFM 40B", "slug": "lfm-40b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Liquid AI", "intelligenceIndex": 5.41798540794277, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0493, "critpt": null, "lcr": null, "gpqa": 0.327272727272727, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": false }, { "name": "MiniMax-M2.5", "shortName": "MiniMax-M2.5", "slug": "minimax-m2-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "intelligenceIndex": 22.7966473221872, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.348484848484849, "scicode": null, "hle": 0.205282669138091, "critpt": 0.0114285714285714, "lcr": 0.733333333333333, "gpqa": 0.848484848484848, "omniscience": -38.86666666666667, "omniscienceAccuracy": 0.2618333333333333, "omniscienceNonHallucination": 0.11876270038383385, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.03, "cacheWritePrice": 0.375, "contextWindowTokens": 204800, "medianOutputTokensPerSecond": 84.5886351514548, "medianTimeToFirstTokenSeconds": 1.47417534250001, "isOpenWeights": true }, { "name": "Mistral Small 4 (Non-reasoning)", "shortName": "Mistral Small 4 (non-reasoning)", "slug": "mistral-small-4-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 8.99008610259603, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.106060606060606, "scicode": null, "hle": 0.0379981464318814, "critpt": 0.00285714285714286, "lcr": 0.283333333333333, "gpqa": 0.570707070707071, "omniscience": -48.516666666666666, "omniscienceAccuracy": 0.16566666666666666, "omniscienceNonHallucination": 0.21993607670795046, "gdpvalNormalized": null, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.6, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 166.564249820339, "medianTimeToFirstTokenSeconds": 0.726275960999999, "isOpenWeights": true }, { "name": "Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)", "shortName": "Claude Opus 5 (xhigh)", "slug": "claude-opus-5-xhigh", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 49.6773552926939, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 4.877843772646787, "terminalBench40": 0.464646464646465, "terminalBench21": 0.880149812734082, "terminalbenchHard": null, "scicode": 0.556712962962963, "hle": 0.544022242817424, "critpt": 0.277142857142857, "lcr": 0.803333333333333, "gpqa": 0.937373737373737, "omniscience": 35.38333333333333, "omniscienceAccuracy": 0.595, "omniscienceNonHallucination": 0.40452674897119345, "gdpvalNormalized": 0.58815, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 50.6948201732173, "medianTimeToFirstTokenSeconds": 45.5588879979998, "isOpenWeights": false }, { "name": "Claude 3.7 Sonnet (Reasoning)", "shortName": "Claude 3.7 Sonnet", "slug": "claude-3-7-sonnet-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 17.7114460194322, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.212121212121212, "scicode": null, "hle": 0.0967, "critpt": 0.00857142857142857, "lcr": null, "gpqa": 0.771717171717172, "omniscience": -0.7333333333333333, "omniscienceAccuracy": 0.2803333333333333, "omniscienceNonHallucination": 0.6002779064381658, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Claude Sonnet 5 (Adaptive Reasoning, Low Effort)", "shortName": "Claude Sonnet 5 (low)", "slug": "claude-sonnet-5-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 24.2638393579597, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5087545600633299, "terminalBench40": 0.0252525252525253, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.501157407407407, "hle": 0.219184430027804, "critpt": 0.0457142857142857, "lcr": 0.673333333333333, "gpqa": null, "omniscience": -8.233333333333333, "omniscienceAccuracy": 0.3735, "omniscienceNonHallucination": 0.2724128757648311, "gdpvalNormalized": 0.279275, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 57.5789379104957, "medianTimeToFirstTokenSeconds": 1.41293507450002, "isOpenWeights": false }, { "name": "MiMo-V2-Flash (Feb 2026)", "shortName": "MiMo-V2-Flash (Feb 2026)", "slug": "mimo-v2-0206", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 22.4406462861731, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.310606060606061, "scicode": null, "hle": 0.221037998146432, "critpt": 0.0285714285714286, "lcr": 0.713333333333333, "gpqa": 0.835353535353535, "omniscience": -19.05, "omniscienceAccuracy": 0.19783333333333333, "omniscienceNonHallucination": 0.5158944525244131, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GLM-5-Turbo", "shortName": "GLM-5-Turbo", "slug": "glm-5-turbo", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 26.5965803042835, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.333333333333333, "scicode": null, "hle": 0.277571825764597, "critpt": 0.00285714285714286, "lcr": 0.716666666666667, "gpqa": 0.847474747474747, "omniscience": -16.366666666666667, "omniscienceAccuracy": 0.2845, "omniscienceNonHallucination": 0.3736314931283484, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Ling-1T", "shortName": "Ling-1T", "slug": "ling-1t", "isReasoning": false, "deprecated": true, "modelCreatorName": "InclusionAI", "intelligenceIndex": 9.17207010042254, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.106060606060606, "scicode": null, "hle": 0.0732159406858202, "critpt": 0.00571428571428571, "lcr": 0.366666666666667, "gpqa": 0.719191919191919, "omniscience": -55.13333333333333, "omniscienceAccuracy": 0.20333333333333334, "omniscienceNonHallucination": 0.052719665271966476, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "gpt-oss-20b (Low)", "shortName": "gpt-oss-20b (low)", "slug": "gpt-oss-20b-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 9.95295342568922, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0528266913809083, "critpt": 0, "lcr": 0.31, "gpqa": 0.611111111111111, "omniscience": -58.55, "omniscienceAccuracy": 0.15083333333333335, "omniscienceNonHallucination": 0.13287536800785082, "gdpvalNormalized": null, "price1mInputTokens": 0.07, "price1mOutputTokens": 0.215, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 198.485576120611, "medianTimeToFirstTokenSeconds": 0.911608262000016, "isOpenWeights": true }, { "name": "Qwen3.5 27B (Reasoning)", "shortName": "Qwen3.5 27B", "slug": "qwen3-5-27b", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 22.8992942688298, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.325757575757576, "scicode": null, "hle": 0.239110287303058, "critpt": 0.00853061224489796, "lcr": 0.776666666666667, "gpqa": 0.857575757575758, "omniscience": -43.983333333333334, "omniscienceAccuracy": 0.20666666666666667, "omniscienceNonHallucination": 0.1850840336134454, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 75.4410777609304, "medianTimeToFirstTokenSeconds": 5.64308175450001, "isOpenWeights": true }, { "name": "o3", "shortName": "o3", "slug": "o3", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.200754789201, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.371212121212121, "scicode": null, "hle": 0.2005, "critpt": 0.0114285714285714, "lcr": 0.746666666666667, "gpqa": 0.827272727272727, "omniscience": -15.55, "omniscienceAccuracy": 0.38566666666666666, "omniscienceNonHallucination": 0.11909929462832336, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 8, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 113.412685226085, "medianTimeToFirstTokenSeconds": 6.38012814199996, "isOpenWeights": false }, { "name": "Gemini 1.5 Pro (May '24)", "shortName": "Gemini 1.5 Pro (May)", "slug": "gemini-1-5-pro-may-2024", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 6.4407131644479, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0348, "critpt": null, "lcr": null, "gpqa": 0.370707070707071, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "K-EXAONE 2.0 0803", "shortName": "K-EXAONE 2.0", "slug": "k-exaone-2-0-0803", "isReasoning": true, "deprecated": false, "modelCreatorName": "LG AI Research", "intelligenceIndex": 19.6904943371003, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.404494382022472, "terminalbenchHard": null, "scicode": 0.420138888888889, "hle": 0.185820203892493, "critpt": 0.00857142857142857, "lcr": 0.6, "gpqa": 0.829292929292929, "omniscience": -6.6, "omniscienceAccuracy": 0.1305, "omniscienceNonHallucination": 0.7740080506037953, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Qwen3 4B (Reasoning)", "shortName": "Qwen3 4B", "slug": "qwen3-4b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.22847163984467, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0442, "critpt": null, "lcr": 0, "gpqa": 0.522222222222222, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "GLM 5.3 Flash", "shortName": "GLM-5.3-Flash", "slug": "glm-5-3-flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "Z AI", "intelligenceIndex": 41.807466113455, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.2532595604307378, "terminalBench40": 0.328282828282828, "terminalBench21": 0.842696629213483, "terminalbenchHard": null, "scicode": 0.516203703703704, "hle": 0.398517145505097, "critpt": 0.154285714285714, "lcr": 0.8, "gpqa": 0.912121212121212, "omniscience": 7.466666666666667, "omniscienceAccuracy": 0.275, "omniscienceNonHallucination": 0.7236781609195402, "gdpvalNormalized": 0.57039, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.026, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 44.8000881348958, "medianTimeToFirstTokenSeconds": 3.312944872, "isOpenWeights": true }, { "name": "GPT-6 Luna (Xhigh)", "shortName": "GPT-6 Luna (xhigh)", "slug": "gpt-6-luna-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 33.8845820217785, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.04208218646439018, "terminalBench40": 0.0808080808080808, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.517361111111111, "hle": 0.343373493975904, "critpt": 0.174285714285714, "lcr": 0.8, "gpqa": null, "omniscience": -1.8333333333333333, "omniscienceAccuracy": 0.44166666666666665, "omniscienceNonHallucination": 0.17611940298507467, "gdpvalNormalized": 0.39838999999999997, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.01, "cacheWritePrice": 0.125, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 131.196936638219, "medianTimeToFirstTokenSeconds": 20.8632337255, "isOpenWeights": false }, { "name": "DeepSeek V3.1 (Reasoning)", "shortName": "DeepSeek V3.1", "slug": "deepseek-v3-1-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 13.4758123489932, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.25, "scicode": null, "hle": 0.142724745134384, "critpt": 0.02, "lcr": 0.566666666666667, "gpqa": 0.778787878787879, "omniscience": -29.583333333333332, "omniscienceAccuracy": 0.29, "omniscienceNonHallucination": 0.17488262910798125, "gdpvalNormalized": null, "price1mInputTokens": 0.58, "price1mOutputTokens": 1.68, "cacheHitPrice": 0.58, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Llama 3.3 Nemotron Super 49B v1 (Reasoning)", "shortName": "Llama 3.3 Nemotron Super 49B", "slug": "llama-3-3-nemotron-super-49b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "NVIDIA", "intelligenceIndex": 8.93313646948331, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0562, "critpt": 0, "lcr": 0.183333333333333, "gpqa": 0.643434343434343, "omniscience": -58.3, "omniscienceAccuracy": 0.171, "omniscienceNonHallucination": 0.09047044632086854, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Step 3.5 Flash 2603", "shortName": "Step 3.5 Flash 2603", "slug": "step-3-5-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "StepFun", "intelligenceIndex": 16.964120603515, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.325757575757576, "scicode": null, "hle": 0.244670991658943, "critpt": 0.0228571428571429, "lcr": 0.63, "gpqa": 0.826262626262626, "omniscience": -44.21666666666667, "omniscienceAccuracy": 0.24616666666666667, "omniscienceNonHallucination": 0.08688923281008176, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.3, "cacheHitPrice": 0.02, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 145.792184628446, "medianTimeToFirstTokenSeconds": 3.37458041499997, "isOpenWeights": false }, { "name": "Mistral Large 2 (Jul '24)", "shortName": "Mistral Large 2 (Jul)", "slug": "mistral-large-2407", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 6.7994617527877, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0295, "critpt": null, "lcr": 0.02, "gpqa": 0.471717171717172, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Solar Pro 4", "shortName": "Solar Pro 4", "slug": "solar-pro4", "isReasoning": true, "deprecated": false, "modelCreatorName": "Upstage", "intelligenceIndex": 28.1541333537469, "intelligenceIndexIsEstimated": true, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.573033707865168, "terminalbenchHard": null, "scicode": 0.445601851851852, "hle": 0.292400370713624, "critpt": 0.0542857142857143, "lcr": 0.74, "gpqa": 0.890909090909091, "omniscience": -0.85, "omniscienceAccuracy": 0.18933333333333333, "omniscienceNonHallucination": 0.7559621710526316, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.06, "cacheWritePrice": null, "contextWindowTokens": 512000, "medianOutputTokensPerSecond": 79.8869206812232, "medianTimeToFirstTokenSeconds": 2.08845874049999, "isOpenWeights": false }, { "name": "DBRX Instruct", "shortName": "DBRX", "slug": "dbrx", "isReasoning": false, "deprecated": true, "modelCreatorName": "Databricks", "intelligenceIndex": 5.28869837981177, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0292, "critpt": null, "lcr": null, "gpqa": 0.331313131313131, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "GLM-4.5 (Reasoning)", "shortName": "GLM-4.5", "slug": "glm-4.5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 12.7680269610265, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.21969696969697, "scicode": null, "hle": 0.129749768303985, "critpt": 0, "lcr": 0.526666666666667, "gpqa": 0.781818181818182, "omniscience": -27.383333333333333, "omniscienceAccuracy": 0.25133333333333335, "omniscienceNonHallucination": 0.29853072128227964, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen2.5 Instruct 32B", "shortName": "Qwen2.5 Instruct 32B", "slug": "qwen2.5-32b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.87156485936671, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0399, "critpt": null, "lcr": null, "gpqa": 0.465656565656566, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 2.5 Flash-Lite (Non-reasoning)", "shortName": "Gemini 2.5 Flash-Lite (non-reasoning)", "slug": "gemini-2-5-flash-lite", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 6.66853092464521, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0366079703429101, "critpt": 0, "lcr": 0.32, "gpqa": 0.473737373737374, "omniscience": -58.78333333333333, "omniscienceAccuracy": 0.15433333333333332, "omniscienceNonHallucination": 0.12238864800946003, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 284.568708872763, "medianTimeToFirstTokenSeconds": 0.328639500500032, "isOpenWeights": false }, { "name": "Hermes 3 - Llama-3.1 70B", "shortName": "Hermes 3 - Llama-3.1 70B", "slug": "hermes-3-llama-3-1-70b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Nous Research", "intelligenceIndex": 5.99680149700574, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0399, "critpt": null, "lcr": null, "gpqa": 0.401010101010101, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.7, "price1mOutputTokens": 0.7, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 28.4040860509649, "medianTimeToFirstTokenSeconds": 2.27743959099996, "isOpenWeights": true }, { "name": "North Mini Code", "shortName": "North Mini Code", "slug": "north-mini-code", "isReasoning": true, "deprecated": false, "modelCreatorName": "Cohere", "intelligenceIndex": 9.91462776590372, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.355805243445693, "terminalbenchHard": 0.310606060606061, "scicode": 0.387731481481481, "hle": 0.110750695088044, "critpt": 0.00285714285714286, "lcr": 0.373333333333333, "gpqa": 0.756565656565657, "omniscience": -48.583333333333336, "omniscienceAccuracy": 0.189, "omniscienceNonHallucination": 0.16789971228935474, "gdpvalNormalized": 0, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 70.2836056259075, "medianTimeToFirstTokenSeconds": 0.42702448450001, "isOpenWeights": true }, { "name": "Sonar Reasoning Pro", "shortName": "Sonar Reasoning Pro", "slug": "sonar-reasoning-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "Perplexity", "intelligenceIndex": 11.8218357445928, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 127000, "isOpenWeights": false }, { "name": "GPT-5.5 (Xhigh)", "shortName": "GPT-5.5 (xhigh)", "slug": "gpt-5-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 38.3556437590607, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.6339796841243297, "terminalBench40": 0.146464646464646, "terminalBench21": 0.842696629213483, "terminalbenchHard": 0.606060606060606, "scicode": 0.55787037037037, "hle": 0.457831325301205, "critpt": 0.271428571428571, "lcr": 0.843333333333333, "gpqa": 0.935353535353535, "omniscience": 20.516666666666666, "omniscienceAccuracy": 0.5795, "omniscienceNonHallucination": 0.109789932619897, "gdpvalNormalized": 0.41794500000000007, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 922000, "medianOutputTokensPerSecond": 90.4614659191824, "medianTimeToFirstTokenSeconds": 48.0554188245002, "isOpenWeights": false }, { "name": "Mistral Medium", "shortName": "Mistral Medium", "slug": "mistral-medium", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.49191510422431, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0351, "critpt": null, "lcr": null, "gpqa": 0.349494949494949, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": false }, { "name": "GPT-6 Astra (Xhigh)", "shortName": "GPT-6 Astra (xhigh)", "slug": "gpt-6-astra-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 52.3863277108782, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.308795912269076, "terminalBench40": 0.595959595959596, "terminalBench21": 0.891385767790262, "terminalbenchHard": null, "scicode": 0.556712962962963, "hle": 0.545875810936052, "critpt": 0.314285714285714, "lcr": 0.8, "gpqa": 0.962626262626263, "omniscience": 43.416666666666664, "omniscienceAccuracy": 0.6185, "omniscienceNonHallucination": 0.5168195718654434, "gdpvalNormalized": 0.50824, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 1, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 47.5600540412565, "medianTimeToFirstTokenSeconds": 142.28102478, "isOpenWeights": false }, { "name": "Nova 2.0 Omni (Non-reasoning)", "shortName": "Nova 2.0 Omni (non-reasoning)", "slug": "nova-2-0-omni", "isReasoning": false, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 8.15087501101863, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0384615384615385, "critpt": 0, "lcr": 0.246666666666667, "gpqa": 0.554545454545454, "omniscience": -64.23333333333333, "omniscienceAccuracy": 0.13233333333333333, "omniscienceNonHallucination": 0.10718401844026126, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Granite 4.0 Micro", "shortName": "Granite 4.0 Micro", "slug": "granite-4-0-micro", "isReasoning": false, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 5.10657830942846, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0500463392029657, "critpt": 0, "lcr": 0.07, "gpqa": 0.336363636363636, "omniscience": -77.53333333333333, "omniscienceAccuracy": 0.09316666666666666, "omniscienceNonHallucination": 0.04227164124241867, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Claude 3 Haiku", "shortName": "Claude 3 Haiku", "slug": "claude-3-haiku", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 5.57842403353134, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0407784986098239, "critpt": 0, "lcr": 0.276666666666667, "gpqa": 0.373737373737374, "omniscience": -48.63333333333333, "omniscienceAccuracy": 0.17633333333333334, "omniscienceNonHallucination": 0.1954674220963173, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 1.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Claude Opus 5 (Adaptive Reasoning, Low Effort)", "shortName": "Claude Opus 5 (low)", "slug": "claude-opus-5-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 39.3543357992936, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.098302324514437, "terminalBench40": 0.262626262626263, "terminalBench21": 0.764044943820225, "terminalbenchHard": null, "scicode": 0.491898148148148, "hle": 0.434198331788693, "critpt": 0.231428571428571, "lcr": 0.813333333333333, "gpqa": 0.888888888888889, "omniscience": 28.55, "omniscienceAccuracy": 0.5595, "omniscienceNonHallucination": 0.3779795686719637, "gdpvalNormalized": 0.39683, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 49.3951520661691, "medianTimeToFirstTokenSeconds": 2.96191884299992, "isOpenWeights": false }, { "name": "Qwen3.5 9B (Non-reasoning)", "shortName": "Qwen3.5 9B (non-reasoning)", "slug": "qwen3-5-9b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 13.2659955355465, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.213483146067416, "terminalbenchHard": 0.181818181818182, "scicode": null, "hle": 0.0936051899907322, "critpt": 0.00571428571428571, "lcr": 0.46, "gpqa": 0.785858585858586, "omniscience": -70.7, "omniscienceAccuracy": 0.13983333333333334, "omniscienceNonHallucination": 0.0155008719240457, "gdpvalNormalized": null, "price1mInputTokens": 0.17, "price1mOutputTokens": 0.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Mistral 7B Instruct", "shortName": "Mistral 7B", "slug": "mistral-7b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.03498213171439, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0455, "critpt": 0, "lcr": 0, "gpqa": 0.176767676767677, "omniscience": -64.91666666666667, "omniscienceAccuracy": 0.09016666666666667, "omniscienceNonHallucination": 0.18739695915002752, "gdpvalNormalized": null, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "Gemini 2.5 Flash (Reasoning)", "shortName": "Gemini 2.5 Flash", "slug": "gemini-2-5-flash-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 13.1060438652288, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.136363636363636, "scicode": null, "hle": 0.121408711770158, "critpt": 0.0114285714285714, "lcr": 0.653333333333333, "gpqa": 0.78989898989899, "omniscience": -29.8, "omniscienceAccuracy": 0.2595, "omniscienceNonHallucination": 0.24713031735313973, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.03, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 211.714594138184, "medianTimeToFirstTokenSeconds": 19.775116386, "isOpenWeights": false }, { "name": "Gemma 3 4B Instruct", "shortName": "Gemma 3 4B", "slug": "gemma-3-4b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.00374531835205993, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0529, "critpt": 0, "lcr": 0.0666666666666667, "gpqa": 0.290909090909091, "omniscience": -82.88333333333334, "omniscienceAccuracy": 0.07733333333333334, "omniscienceNonHallucination": 0.017882947976878616, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3.5 122B A10B (Reasoning)", "shortName": "Qwen3.5 122B A10B", "slug": "qwen3-5-122b-a10b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 15.5694009312969, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.3247468637357143, "terminalBench40": 0, "terminalBench21": 0.47565543071161, "terminalbenchHard": 0.310606060606061, "scicode": 0.396990740740741, "hle": 0.252085264133457, "critpt": 0.00571428571428571, "lcr": 0.763333333333333, "gpqa": 0.856565656565657, "omniscience": -41.5, "omniscienceAccuracy": 0.24366666666666667, "omniscienceNonHallucination": 0.129131776112825, "gdpvalNormalized": 0.15089999999999998, "price1mInputTokens": 0.4, "price1mOutputTokens": 3.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 124.849032405179, "medianTimeToFirstTokenSeconds": 2.42432966549998, "isOpenWeights": true }, { "name": "MiMo-V2.5", "shortName": "MiMo-V2.5", "slug": "mimo-v2-5-0424", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 25.1743063555572, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.636704119850187, "terminalbenchHard": 0.416666666666667, "scicode": 0.438657407407407, "hle": 0.272011121408712, "critpt": 0.0371428571428571, "lcr": 0.73, "gpqa": 0.84949494949495, "omniscience": -9.833333333333334, "omniscienceAccuracy": 0.1675, "omniscienceNonHallucination": 0.6806806806806807, "gdpvalNormalized": 0.24310500000000002, "price1mInputTokens": 0.14, "price1mOutputTokens": 0.28, "cacheHitPrice": 0.0028, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 44.5305638678603, "medianTimeToFirstTokenSeconds": 13.9020522289999, "isOpenWeights": true }, { "name": "Grok 4.1 Fast (Reasoning)", "shortName": "Grok 4.1 Fast", "slug": "grok-4-1-fast-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 20.3701852131578, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.242424242424242, "scicode": null, "hle": 0.193234476367006, "critpt": 0.0285714285714286, "lcr": 0.74, "gpqa": 0.852525252525253, "omniscience": -29.916666666666668, "omniscienceAccuracy": 0.25083333333333335, "omniscienceNonHallucination": 0.26585094549499444, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "Hermes 4 - Llama-3.1 70B (Non-reasoning)", "shortName": "Hermes 4 70B (non-reasoning)", "slug": "hermes-4-llama-3-1-70b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Nous Research", "intelligenceIndex": 6.65822068462287, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0356811862835959, "critpt": 0, "lcr": 0.0466666666666667, "gpqa": 0.490909090909091, "omniscience": -48.583333333333336, "omniscienceAccuracy": 0.18666666666666668, "omniscienceNonHallucination": 0.173155737704918, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "NVIDIA Nemotron 3 Nano 30B A3B (Non-reasoning)", "shortName": "Nemotron 3 Nano (non-reasoning)", "slug": "nvidia-nemotron-3-nano-30b-a3b", "isReasoning": false, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 6.8478519055181, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.121212121212121, "scicode": null, "hle": 0.0458758109360519, "critpt": 0, "lcr": 0.106666666666667, "gpqa": 0.398989898989899, "omniscience": -69.3, "omniscienceAccuracy": 0.11366666666666667, "omniscienceNonHallucination": 0.08988341481760065, "gdpvalNormalized": 0, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.2, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 231.273913603179, "medianTimeToFirstTokenSeconds": 0.695761350499935, "isOpenWeights": true }, { "name": "Claude Sonnet 5 (Adaptive Reasoning, Max Effort)", "shortName": "Claude Sonnet 5 (max)", "slug": "claude-sonnet-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 38.1638712882576, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 5.091163584815694, "terminalBench40": 0.141414141414141, "terminalBench21": 0.805243445692884, "terminalbenchHard": null, "scicode": 0.542824074074074, "hle": 0.412882298424467, "critpt": 0.168571428571429, "lcr": 0.82, "gpqa": 0.911111111111111, "omniscience": 16.45, "omniscienceAccuracy": 0.4005, "omniscienceNonHallucination": 0.6063386155129274, "gdpvalNormalized": 0.4746, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 75.7016311676748, "medianTimeToFirstTokenSeconds": 197.5922349405, "isOpenWeights": false }, { "name": "Grok 3", "shortName": "Grok 3", "slug": "grok-3", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 12.1076211741685, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.113636363636364, "scicode": null, "hle": 0.0412, "critpt": 0, "lcr": 0.58, "gpqa": 0.692929292929293, "omniscience": -33.18333333333333, "omniscienceAccuracy": 0.289, "omniscienceNonHallucination": 0.12681669010782937, "gdpvalNormalized": null, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 1.5375, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "GLM 5V Turbo (Reasoning)", "shortName": "GLM 5V Turbo", "slug": "glm-5v-turbo", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 23.4967112022207, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.325757575757576, "scicode": null, "hle": 0.171455050973123, "critpt": 0.00571428571428571, "lcr": 0.703333333333333, "gpqa": 0.809090909090909, "omniscience": -19.283333333333335, "omniscienceAccuracy": 0.29333333333333333, "omniscienceNonHallucination": 0.3120283018867924, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Mistral Small 3.1", "shortName": "Mistral Small 3.1", "slug": "mistral-small-3-1", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 7.11735622656558, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.03502860053476431, "terminalBench40": 0, "terminalBench21": 0.262172284644195, "terminalbenchHard": 0.0757575757575758, "scicode": 0.277777777777778, "hle": 0.0430954587581094, "critpt": 0, "lcr": 0.223333333333333, "gpqa": 0.453535353535354, "omniscience": -50.75, "omniscienceAccuracy": 0.151, "omniscienceNonHallucination": 0.22438162544169615, "gdpvalNormalized": 0, "price1mInputTokens": 0.11, "price1mOutputTokens": 0.17, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "MiMo-V2.6-Pro", "shortName": "MiMo-V2.6-Pro", "slug": "mimo-v2-6-pro", "isReasoning": true, "deprecated": false, "modelCreatorName": "Xiaomi", "intelligenceIndex": 46.3242065310383, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.13322318937213493, "terminalBench40": 0.348484848484849, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.608796296296296, "hle": 0.493512511584801, "critpt": 0.265714285714286, "lcr": 0.863333333333333, "gpqa": null, "omniscience": 8.383333333333333, "omniscienceAccuracy": 0.3485, "omniscienceNonHallucination": 0.5937579943719622, "gdpvalNormalized": 0.589145, "price1mInputTokens": 0.435, "price1mOutputTokens": 0.87, "cacheHitPrice": 0.0036, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 41.7029509194572, "medianTimeToFirstTokenSeconds": 4.204969006, "isOpenWeights": true }, { "name": "Claude Opus 4.5 (Reasoning)", "shortName": "Claude Opus 4.5", "slug": "claude-opus-4-5-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 29.0955143334264, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.46969696969697, "scicode": null, "hle": 0.301204819277108, "critpt": 0.0457142857142857, "lcr": 0.773333333333333, "gpqa": 0.865656565656566, "omniscience": 14, "omniscienceAccuracy": 0.4658333333333333, "omniscienceNonHallucination": 0.39001560062402496, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 47.1977807122778, "medianTimeToFirstTokenSeconds": 15.316523623, "isOpenWeights": false }, { "name": "Grok 4.6 (Medium)", "shortName": "Grok 4.6 (medium)", "slug": "grok-4-6-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 42.836322654063, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.4963233956114492, "terminalBench40": 0.131313131313131, "terminalBench21": 0.842696629213483, "terminalbenchHard": null, "scicode": 0.559027777777778, "hle": 0.421223354958295, "critpt": 0.177142857142857, "lcr": 0.81, "gpqa": 0.935353535353535, "omniscience": 28, "omniscienceAccuracy": 0.41933333333333334, "omniscienceNonHallucination": 0.7600459242250287, "gdpvalNormalized": 0.5524199999999999, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 68.4560921073166, "medianTimeToFirstTokenSeconds": 24.8065508735, "isOpenWeights": false }, { "name": "Claude Sonnet 5.5 (Adaptive Reasoning, High Effort, Default Fallback)", "shortName": "Claude Sonnet 5.5 (high with fallback)", "slug": "claude-sonnet-5-5-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 46.7356921799024, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.079758672715514, "terminalBench40": 0.439393939393939, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.537037037037037, "hle": 0.458294717330862, "critpt": 0.245714285714286, "lcr": 0.78, "gpqa": null, "omniscience": 20.95, "omniscienceAccuracy": 0.5198333333333334, "omniscienceNonHallucination": 0.3536966331135023, "gdpvalNormalized": 0.5086649999999999, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 93.5002270462924, "medianTimeToFirstTokenSeconds": 12.8820387214999, "isOpenWeights": false }, { "name": "Qwen3 Next 80B A3B Instruct", "shortName": "Qwen3 Next 80B A3B", "slug": "qwen3-next-80b-a3b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.64335007559183, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0757575757575758, "scicode": null, "hle": 0.0764596848934198, "critpt": 0, "lcr": 0.526666666666667, "gpqa": 0.738383838383838, "omniscience": -59.5, "omniscienceAccuracy": 0.1725, "omniscienceNonHallucination": 0.07250755287009059, "gdpvalNormalized": null, "price1mInputTokens": 0.15, "price1mOutputTokens": 1.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 165.346410836059, "medianTimeToFirstTokenSeconds": 2.19224850399996, "isOpenWeights": true }, { "name": "Llama Nemotron Super 49B v1.5 (Non-reasoning)", "shortName": "Llama Nemotron Super 49B v1.5 (non-reasoning)", "slug": "llama-nemotron-super-49b-v1-5", "isReasoning": false, "deprecated": true, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.36435496013793, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0430954587581094, "critpt": 0, "lcr": 0.246666666666667, "gpqa": 0.480808080808081, "omniscience": -46.18333333333333, "omniscienceAccuracy": 0.126, "omniscienceNonHallucination": 0.32742181540808546, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-5.6 Terra (Medium)", "shortName": "GPT-5.6 Terra (medium)", "slug": "gpt-5-6-terra-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 30.0938023419377, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.18337295992969213, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.722846441947566, "terminalbenchHard": null, "scicode": 0.50462962962963, "hle": 0.332715477293791, "critpt": 0.174285714285714, "lcr": 0.74, "gpqa": 0.871717171717172, "omniscience": -5.133333333333334, "omniscienceAccuracy": 0.446, "omniscienceNonHallucination": 0.10228640192539107, "gdpvalNormalized": 0.37730499999999995, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 73.5206998419488, "medianTimeToFirstTokenSeconds": 1.9536432175, "isOpenWeights": false }, { "name": "Gemini 2.5 Pro", "shortName": "Gemini 2.5 Pro", "slug": "gemini-2-5-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 16.0778962941864, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.23166603200256353, "terminalBench40": 0, "terminalBench21": 0.284644194756554, "terminalbenchHard": 0.265151515151515, "scicode": 0.462962962962963, "hle": 0.225208526413346, "critpt": 0.0257142857142857, "lcr": 0.69, "gpqa": 0.844444444444444, "omniscience": -16.35, "omniscienceAccuracy": 0.3905, "omniscienceNonHallucination": 0.0910582444626743, "gdpvalNormalized": 0, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 120.150826840778, "medianTimeToFirstTokenSeconds": 23.516249387, "isOpenWeights": false }, { "name": "Grok 4.7 (High)", "shortName": "Grok 4.7 (high)", "slug": "grok-4-7-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 46.3321770885625, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.726106691786027, "terminalBench40": 0.247474747474747, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.577546296296296, "hle": 0.423076923076923, "critpt": 0.18, "lcr": 0.77, "gpqa": null, "omniscience": 30.9, "omniscienceAccuracy": 0.4781666666666667, "omniscienceNonHallucination": 0.6758224209517726, "gdpvalNormalized": 0.5968600000000001, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 75.8229717391692, "medianTimeToFirstTokenSeconds": 49.4087205159999, "isOpenWeights": false }, { "name": "MiniMax-M2.1", "shortName": "MiniMax-M2.1", "slug": "minimax-m2-1", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "intelligenceIndex": 20.9452850207622, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.287878787878788, "scicode": null, "hle": 0.231696014828545, "critpt": 0.00285714285714286, "lcr": 0.676666666666667, "gpqa": 0.83030303030303, "omniscience": -32.86666666666667, "omniscienceAccuracy": 0.21166666666666667, "omniscienceNonHallucination": 0.31458773784355176, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.03, "cacheWritePrice": 0.375, "contextWindowTokens": 204800, "medianOutputTokensPerSecond": 87.3396233417361, "medianTimeToFirstTokenSeconds": 1.51171361249994, "isOpenWeights": true }, { "name": "MiniMax-M3", "shortName": "MiniMax-M3", "slug": "minimax-m3", "isReasoning": true, "deprecated": false, "modelCreatorName": "MiniMax", "intelligenceIndex": 29.2202932236316, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5076126905781733, "terminalBench40": 0.0202020202020202, "terminalBench21": 0.651685393258427, "terminalbenchHard": 0.424242424242424, "scicode": 0.471064814814815, "hle": 0.389712696941613, "critpt": 0.0371428571428571, "lcr": 0.83, "gpqa": 0.929292929292929, "omniscience": 1.35, "omniscienceAccuracy": 0.167, "omniscienceNonHallucination": 0.8157262905162065, "gdpvalNormalized": 0.37291499999999994, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.06, "cacheWritePrice": 0.375, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 99.3894365235475, "medianTimeToFirstTokenSeconds": 1.21466316799999, "isOpenWeights": true }, { "name": "Grok 2 (Dec '24)", "shortName": "Grok 2", "slug": "grok-2-1212", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 7.10559707757169, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0306, "critpt": null, "lcr": null, "gpqa": 0.51010101010101, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "MiMo-V2-Omni-0327", "shortName": "MiMo-V2-Omni-0327", "slug": "mimo-v2-omni-0327", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 25.1205269950478, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.356060606060606, "scicode": null, "hle": 0.225671918443003, "critpt": 0.00857142857142857, "lcr": 0.733333333333333, "gpqa": 0.854545454545455, "omniscience": -15.1, "omniscienceAccuracy": 0.17816666666666667, "omniscienceNonHallucination": 0.5994727235854795, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "GPT-5 mini (High)", "shortName": "GPT-5 mini (high)", "slug": "gpt-5-mini", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 16.7730881042515, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.05349965791130692, "terminalBench40": 0, "terminalBench21": 0.0374531835205993, "terminalbenchHard": 0.333333333333333, "scicode": 0.390046296296296, "hle": 0.214550509731233, "critpt": 0, "lcr": 0.723333333333333, "gpqa": 0.828282828282828, "omniscience": -17.316666666666666, "omniscienceAccuracy": 0.24966666666666668, "omniscienceNonHallucination": 0.4364726788094181, "gdpvalNormalized": 0.12694, "price1mInputTokens": 0.25, "price1mOutputTokens": 2, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 118.756923937273, "medianTimeToFirstTokenSeconds": 58.987530249, "isOpenWeights": false }, { "name": "Nemotron 3.5 Lightning", "shortName": "Nemotron 3.5 Lightning", "slug": "nemotron-3-5-lightning", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 12.8572306217144, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.09309918417597017, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.243445692883895, "terminalbenchHard": null, "scicode": 0.320601851851852, "hle": 0.105653382761817, "critpt": 0, "lcr": 0.603333333333333, "gpqa": 0.743434343434343, "omniscience": -17.716666666666665, "omniscienceAccuracy": 0.14433333333333334, "omniscienceNonHallucination": 0.6242695753798209, "gdpvalNormalized": 0.06211000000000001, "price1mInputTokens": 0.06, "price1mOutputTokens": 0.2, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 302.45449329289, "medianTimeToFirstTokenSeconds": 0.577816639499929, "isOpenWeights": true }, { "name": "Qwen3 4B 2507 (Reasoning)", "shortName": "Qwen3 4B 2507", "slug": "qwen3-4b-2507-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.79842923104994, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.061631139944393, "critpt": 0, "lcr": 0.373333333333333, "gpqa": 0.666666666666667, "omniscience": -60.266666666666666, "omniscienceAccuracy": 0.12533333333333332, "omniscienceNonHallucination": 0.16768292682926833, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Gemini 1.5 Pro (Sep '24)", "shortName": "Gemini 1.5 Pro (Sep)", "slug": "gemini-1-5-pro", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 7.90554657851904, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0464, "critpt": null, "lcr": null, "gpqa": 0.588888888888889, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "Gemini 1.0 Ultra", "shortName": "Gemini 1.0 Ultra", "slug": "gemini-1-0-ultra", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 5.83837664836323, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": false }, { "name": "Qwen3 1.7B (Non-reasoning)", "shortName": "Qwen3 1.7B (non-reasoning)", "slug": "qwen3-1.7b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 4.8568135993386, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0532, "critpt": 0, "lcr": 0, "gpqa": 0.282828282828283, "omniscience": -84.08333333333333, "omniscienceAccuracy": 0.06916666666666667, "omniscienceNonHallucination": 0.022381378692927445, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Gemma 3 27B Instruct", "shortName": "Gemma 3 27B", "slug": "gemma-3-27b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 4.85297805617922, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0, "terminalBench40": 0, "terminalBench21": 0.0449438202247191, "terminalbenchHard": 0.0378787878787879, "scicode": 0.232638888888889, "hle": 0.044, "critpt": 0, "lcr": 0.0733333333333333, "gpqa": 0.428282828282828, "omniscience": -67.21666666666667, "omniscienceAccuracy": 0.1295, "omniscienceNonHallucination": 0.07907332950411639, "gdpvalNormalized": 0, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GLM-4.7-Flash (Reasoning)", "shortName": "GLM-4.7-Flash", "slug": "glm-4-7-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 14.8719733920138, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.21969696969697, "scicode": null, "hle": 0.0759962928637627, "critpt": 0.00285714285714286, "lcr": 0.416666666666667, "gpqa": 0.580808080808081, "omniscience": -62.61666666666667, "omniscienceAccuracy": 0.1615, "omniscienceNonHallucination": 0.06062413039157222, "gdpvalNormalized": null, "price1mInputTokens": 0.07, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 60.4130689461287, "medianTimeToFirstTokenSeconds": 1.47417420300008, "isOpenWeights": true }, { "name": "Phi-4 Multimodal Instruct", "shortName": "Phi-4 Multimodal", "slug": "phi-4-multimodal", "isReasoning": false, "deprecated": false, "modelCreatorName": "Microsoft", "intelligenceIndex": 5.80520041934726, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.05, "critpt": null, "lcr": null, "gpqa": 0.315151515151515, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Granite 3.3 8B (Non-reasoning)", "shortName": "Granite 3.3 8B (non-reasoning)", "slug": "granite-3-3-8b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "IBM", "intelligenceIndex": 4.92158293624767, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0417052826691381, "critpt": 0, "lcr": null, "gpqa": 0.338383838383838, "omniscience": -77.28333333333333, "omniscienceAccuracy": 0.0965, "omniscienceNonHallucination": 0.03781590112525368, "gdpvalNormalized": null, "price1mInputTokens": 0.03, "price1mOutputTokens": 0.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 10.1809447885661, "medianTimeToFirstTokenSeconds": 42.137543117, "isOpenWeights": true }, { "name": "DeepSeek-V2-Chat", "shortName": "DeepSeek-V2", "slug": "deepseek-v2", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 5.50606445861584, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3.5 9B (Reasoning)", "shortName": "Qwen3.5 9B", "slug": "qwen3-5-9b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 11.1961650827156, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.21258346272934256, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.292134831460674, "terminalbenchHard": 0.242424242424242, "scicode": 0.295138888888889, "hle": 0.149212233549583, "critpt": 0.00285714285714286, "lcr": 0.7, "gpqa": 0.806060606060606, "omniscience": -53.46666666666667, "omniscienceAccuracy": 0.16416666666666666, "omniscienceNonHallucination": 0.1639082751744766, "gdpvalNormalized": 0, "price1mInputTokens": 0.135, "price1mOutputTokens": 0.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 33.7308754347155, "medianTimeToFirstTokenSeconds": 2.42126639750001, "isOpenWeights": true }, { "name": "o3-mini", "shortName": "o3-mini", "slug": "o3-mini", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 12.4707708274399, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0791, "critpt": null, "lcr": null, "gpqa": 0.748484848484848, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 1.1, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.55, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 200.272348022687, "medianTimeToFirstTokenSeconds": 6.6040161274999, "isOpenWeights": false }, { "name": "Grok 4 Fast (Non-reasoning)", "shortName": "Grok 4 Fast (non-reasoning)", "slug": "grok-4-fast", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 11.0641868989941, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.121212121212121, "scicode": null, "hle": 0.0454124189063948, "critpt": 0, "lcr": 0.24, "gpqa": 0.606060606060606, "omniscience": -54.43333333333333, "omniscienceAccuracy": 0.177, "omniscienceNonHallucination": 0.12353179424868366, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "Claude Sonnet 4.6 (Non-reasoning, High Effort)", "shortName": "Claude Sonnet 4.6 (non-reasoning, high)", "slug": "claude-sonnet-4-6", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 24.6869297210126, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.462121212121212, "scicode": null, "hle": 0.133456904541242, "critpt": 0.00857142857142857, "lcr": 0.683333333333333, "gpqa": 0.798989898989899, "omniscience": -3.55, "omniscienceAccuracy": 0.3855, "omniscienceNonHallucination": 0.3148901545972336, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 43.1542384827049, "medianTimeToFirstTokenSeconds": 1.367963288, "isOpenWeights": false }, { "name": "Ling-2.6-1T", "shortName": "Ling-2.6-1T", "slug": "ling-2-6-1t", "isReasoning": false, "deprecated": true, "modelCreatorName": "InclusionAI", "intelligenceIndex": 17.0002388154878, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.310606060606061, "scicode": null, "hle": 0.0871177015755329, "critpt": 0.00285714285714286, "lcr": 0.416666666666667, "gpqa": 0.751515151515151, "omniscience": -50.6, "omniscienceAccuracy": 0.219, "omniscienceNonHallucination": 0.07170294494238161, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Gemma 3 12B Instruct", "shortName": "Gemma 3 12B", "slug": "gemma-3-12b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 3.7508042118297, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0, "terminalBench40": 0, "terminalBench21": 0, "terminalbenchHard": 0.00757575757575758, "scicode": 0.164351851851852, "hle": 0.0421, "critpt": 0, "lcr": 0.0833333333333333, "gpqa": 0.349494949494949, "omniscience": -77.06666666666666, "omniscienceAccuracy": 0.103, "omniscienceNonHallucination": 0.026012634708286853, "gdpvalNormalized": 0, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "NVIDIA Nemotron Nano 9B V2 (Non-reasoning)", "shortName": "NVIDIA Nemotron Nano 9B V2 (non-reasoning)", "slug": "nvidia-nemotron-nano-9b-v2", "isReasoning": false, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 6.84412514839364, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0458758109360519, "critpt": 0, "lcr": 0.24, "gpqa": 0.556565656565657, "omniscience": -57.25, "omniscienceAccuracy": 0.09816666666666667, "omniscienceNonHallucination": 0.2563296987617816, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.195, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 150.329381119629, "medianTimeToFirstTokenSeconds": 2.24300587899999, "isOpenWeights": true }, { "name": "Command A+", "shortName": "Command A+", "slug": "command-a-plus", "isReasoning": true, "deprecated": false, "modelCreatorName": "Cohere", "intelligenceIndex": 13.1335002685349, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.228464419475655, "terminalbenchHard": 0.25, "scicode": 0.385416666666667, "hle": 0.119555143651529, "critpt": 0.00285714285714286, "lcr": 0.526666666666667, "gpqa": 0.760606060606061, "omniscience": -4.016666666666667, "omniscienceAccuracy": 0.08883333333333333, "omniscienceNonHallucination": 0.8584232668739711, "gdpvalNormalized": 1.4999999999986358e-05, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 192000, "medianOutputTokensPerSecond": 210.680051857601, "medianTimeToFirstTokenSeconds": 0.534611388499989, "isOpenWeights": true }, { "name": "GPT-6 Astra (Max)", "shortName": "GPT-6 Astra (max)", "slug": "gpt-6-astra", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 52.673669395513, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 3.2575003134834164, "terminalBench40": 0.590909090909091, "terminalBench21": 0.883895131086142, "terminalbenchHard": null, "scicode": 0.564814814814815, "hle": 0.546802594995366, "critpt": 0.317142857142857, "lcr": 0.806666666666667, "gpqa": 0.960606060606061, "omniscience": 43.4, "omniscienceAccuracy": 0.626, "omniscienceNonHallucination": 0.4866310160427807, "gdpvalNormalized": 0.5209450000000001, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 1, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 51.2323983823932, "medianTimeToFirstTokenSeconds": 320.270991823, "isOpenWeights": false }, { "name": "Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)", "shortName": "Claude Opus 5.5 (max with fallback)", "slug": "claude-opus-5-5", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 57.6223698102963, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 5.982012019521066, "terminalBench40": 0.595959595959596, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.668981481481482, "hle": 0.613531047265987, "critpt": 0.317142857142857, "lcr": 0.846666666666667, "gpqa": null, "omniscience": 46.416666666666664, "omniscienceAccuracy": 0.6621666666666667, "omniscienceNonHallucination": 0.41391218549580666, "gdpvalNormalized": 0.673085, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.2, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 92.1589239077081, "medianTimeToFirstTokenSeconds": 702.554261306, "isOpenWeights": false }, { "name": "EXAONE 4.0 32B (Non-reasoning)", "shortName": "EXAONE 4.0 32B (non-reasoning)", "slug": "exaone-4-0-32b", "isReasoning": false, "deprecated": false, "modelCreatorName": "LG AI Research", "intelligenceIndex": 6.32621718534092, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0500463392029657, "critpt": 0, "lcr": null, "gpqa": 0.628282828282828, "omniscience": -62.766666666666666, "omniscienceAccuracy": 0.10616666666666667, "omniscienceNonHallucination": 0.17900428864441542, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131000, "isOpenWeights": true }, { "name": "Claude Opus 4.7 (Non-reasoning, High Effort)", "shortName": "Claude Opus 4.7 (non-reasoning, high)", "slug": "claude-opus-4-7-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 30.9316777597043, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.545454545454545, "scicode": null, "hle": 0.333178869323448, "critpt": 0.0514285714285714, "lcr": 0.756666666666667, "gpqa": 0.884848484848485, "omniscience": 14.833333333333334, "omniscienceAccuracy": 0.44733333333333336, "omniscienceNonHallucination": 0.45898673100120624, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 41.984005475127, "medianTimeToFirstTokenSeconds": 1.23456539000012, "isOpenWeights": false }, { "name": "G9v3-3B", "shortName": "G9v3-3B", "slug": "g9v3-3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "AI9Stars", "intelligenceIndex": 10.841600440564, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0599250936329588, "terminalbenchHard": null, "scicode": null, "hle": 0.0449490268767377, "critpt": 0, "lcr": 0.413333333333333, "gpqa": 0.438383838383838, "omniscience": -4.366666666666666, "omniscienceAccuracy": 0.06533333333333333, "omniscienceNonHallucination": 0.8833808844507846, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "GLM-4.5V (Reasoning)", "shortName": "GLM-4.5V", "slug": "glm-4-5v-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 7.55612311182388, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.053030303030303, "scicode": null, "hle": 0.0630213160333642, "critpt": 0, "lcr": 0, "gpqa": 0.683838383838384, "omniscience": -46.28333333333333, "omniscienceAccuracy": 0.20833333333333334, "omniscienceNonHallucination": 0.15221052631578946, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 1.8, "cacheHitPrice": 0.11, "cacheWritePrice": null, "contextWindowTokens": 64000, "medianOutputTokensPerSecond": 44.8605243719045, "medianTimeToFirstTokenSeconds": 2.75907078499995, "isOpenWeights": true }, { "name": "Qwen3.5 4B (Non-reasoning)", "shortName": "Qwen3.5 4B (non-reasoning)", "slug": "qwen3-5-4b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 10.812307683746, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.213483146067416, "terminalbenchHard": 0.113636363636364, "scicode": null, "hle": 0.0797034291010195, "critpt": 0, "lcr": 0.346666666666667, "gpqa": 0.712121212121212, "omniscience": -74.73333333333333, "omniscienceAccuracy": 0.11216666666666666, "omniscienceNonHallucination": 0.031912896564670556, "gdpvalNormalized": null, "price1mInputTokens": 0.03, "price1mOutputTokens": 0.15, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 16.2685397847332, "medianTimeToFirstTokenSeconds": 0.938502635500072, "isOpenWeights": true }, { "name": "Qwen3.5 397B A17B (Non-reasoning)", "shortName": "Qwen3.5 397B A17B (non-reasoning)", "slug": "qwen3-5-397b-a17b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 21.4366547759392, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.356060606060606, "scicode": null, "hle": 0.198331788693234, "critpt": 0.00857142857142857, "lcr": 0.643333333333333, "gpqa": 0.860606060606061, "omniscience": -37.9, "omniscienceAccuracy": 0.24516666666666667, "omniscienceNonHallucination": 0.1731066460587326, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 3.6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 86.3679379478889, "medianTimeToFirstTokenSeconds": 2.304392715, "isOpenWeights": true }, { "name": "K2 Horizon 3.7B", "shortName": "K2 Horizon 3.7B", "slug": "k2-horizon-3-7b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 15.6103951330976, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.280898876404494, "terminalbenchHard": null, "scicode": 0.219907407407407, "hle": 0.13855421686747, "critpt": 0, "lcr": 0.623333333333333, "gpqa": 0.691919191919192, "omniscience": -28.616666666666667, "omniscienceAccuracy": 0.10766666666666666, "omniscienceNonHallucination": 0.558647740007471, "gdpvalNormalized": 0.18229, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 524288, "isOpenWeights": true }, { "name": "Qwen3 235B A22B 2507 Instruct", "shortName": "Qwen3 235B 2507", "slug": "qwen3-235b-a22b-instruct-2507", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 11.9970837221274, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.151515151515152, "scicode": null, "hle": 0.111214087117702, "critpt": 0, "lcr": 0.339, "gpqa": 0.752525252525253, "omniscience": -43.983333333333334, "omniscienceAccuracy": 0.18733333333333332, "omniscienceNonHallucination": 0.2282608695652174, "gdpvalNormalized": null, "price1mInputTokens": 0.23, "price1mOutputTokens": 0.92, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 58.2228770514555, "medianTimeToFirstTokenSeconds": 2.4695099755, "isOpenWeights": true }, { "name": "Olmo 3 7B Think", "shortName": "Olmo 3 7B Think", "slug": "olmo-3-7b-think", "isReasoning": true, "deprecated": false, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 5.61691854830355, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0597775718257646, "critpt": 0, "lcr": 0, "gpqa": 0.516161616161616, "omniscience": -71.8, "omniscienceAccuracy": 0.11083333333333334, "omniscienceNonHallucination": 0.0678537956888472, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "HyperCLOVA X SEED Think (32B)", "shortName": "HyperCLOVA X SEED Think (32B)", "slug": "hyperclova-x-seed-think-32b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Naver", "intelligenceIndex": 11.3648115832519, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.121212121212121, "scicode": null, "hle": 0.0546802594995366, "critpt": 0, "lcr": 0.136666666666667, "gpqa": 0.615151515151515, "omniscience": -53.28333333333333, "omniscienceAccuracy": 0.154, "omniscienceNonHallucination": 0.18814026792750194, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "HyperNova 60B 2605 (High, Based on gpt-oss-120b)", "shortName": "HyperNova 60B 2605 (high)", "slug": "hypernova-60b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Multiverse Computing", "intelligenceIndex": 11.7380211033452, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.183520599250936, "terminalbenchHard": 0.234848484848485, "scicode": null, "hle": 0.166357738646895, "critpt": 0.00857142857142857, "lcr": 0.363333333333333, "gpqa": 0.733333333333333, "omniscience": -54.583333333333336, "omniscienceAccuracy": 0.15216666666666667, "omniscienceNonHallucination": 0.17672498525653624, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Sonar Pro", "shortName": "Sonar Pro", "slug": "sonar-pro", "isReasoning": false, "deprecated": true, "modelCreatorName": "Perplexity", "intelligenceIndex": 7.60691968274841, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0655, "critpt": null, "lcr": null, "gpqa": 0.577777777777778, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Nova Micro", "shortName": "Nova Micro", "slug": "nova-micro", "isReasoning": false, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 5.8765232381273, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.046339202965709, "critpt": 0, "lcr": 0.13, "gpqa": 0.357575757575758, "omniscience": -48.516666666666666, "omniscienceAccuracy": 0.10033333333333333, "omniscienceNonHallucination": 0.34920340866987776, "gdpvalNormalized": null, "price1mInputTokens": 0.035, "price1mOutputTokens": 0.14, "cacheHitPrice": 0.00875, "cacheWritePrice": null, "contextWindowTokens": 130000, "medianOutputTokensPerSecond": 248.609253155123, "medianTimeToFirstTokenSeconds": 0.873194964499987, "isOpenWeights": false }, { "name": "Sarvam M (Reasoning, Based on Mistral Small 3.1)", "shortName": "Sarvam M", "slug": "sarvam-m-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Sarvam", "intelligenceIndex": 5.31069666559541, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.031047265987025, "critpt": 0, "lcr": 0, "gpqa": 0.416161616161616, "omniscience": -61.6, "omniscienceAccuracy": 0.15233333333333332, "omniscienceNonHallucination": 0.09359024773889102, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Qwen2.5 Turbo", "shortName": "Qwen2.5 Turbo", "slug": "qwen-turbo", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.43324005500213, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0409, "critpt": null, "lcr": null, "gpqa": 0.41010101010101, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.2, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 109.219740334164, "medianTimeToFirstTokenSeconds": 2.20692066450004, "isOpenWeights": false }, { "name": "Qwen3.8 27B (Non-reasoning)", "shortName": "Qwen3.8 27B (non-reasoning)", "slug": "qwen3-8-27b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 20.1502427055595, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.487757391889078, "terminalBench40": 0, "terminalBench21": 0.49063670411985, "terminalbenchHard": null, "scicode": 0.362268518518519, "hle": 0.121408711770158, "critpt": 0.00285714285714286, "lcr": 0.693333333333333, "gpqa": 0.818181818181818, "omniscience": -7.95, "omniscienceAccuracy": 0.08583333333333333, "omniscienceNonHallucination": 0.8191431175934366, "gdpvalNormalized": 0.27610500000000004, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.1, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 50.2090250994468, "medianTimeToFirstTokenSeconds": 3.86844519400006, "isOpenWeights": true }, { "name": "Mistral Small (Feb '24)", "shortName": "Mistral Small (Feb)", "slug": "mistral-small-2402", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.50039200868795, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0409, "critpt": null, "lcr": null, "gpqa": 0.302020202020202, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": false }, { "name": "GPT-6 Luna (Medium)", "shortName": "GPT-6 Luna (medium)", "slug": "gpt-6-luna-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 29.4619523515762, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.01756122553820397, "terminalBench40": 0.0252525252525253, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.509259259259259, "hle": 0.282669138090825, "critpt": 0.105714285714286, "lcr": 0.783333333333333, "gpqa": null, "omniscience": -5.05, "omniscienceAccuracy": 0.43133333333333335, "omniscienceNonHallucination": 0.15269636576787804, "gdpvalNormalized": 0.35915499999999995, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.01, "cacheWritePrice": 0.125, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "INTELLECT-3 (Based on GLM-4.5-Air)", "shortName": "INTELLECT-3", "slug": "intellect-3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Prime Intellect", "intelligenceIndex": 10.6061102858178, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0909090909090909, "scicode": null, "hle": 0.130676552363299, "critpt": 0.00285714285714286, "lcr": 0.396666666666667, "gpqa": 0.760606060606061, "omniscience": -50.85, "omniscienceAccuracy": 0.19583333333333333, "omniscienceNonHallucination": 0.12414507772020722, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131100, "isOpenWeights": true }, { "name": "gpt-oss-20b (High)", "shortName": "gpt-oss-20b (high)", "slug": "gpt-oss-20b", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 8.9675171856126, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.012460640242179213, "terminalBench40": 0, "terminalBench21": 0.138576779026217, "terminalbenchHard": 0.106060606060606, "scicode": 0.388888888888889, "hle": 0.10982391102873, "critpt": 0.0142857142857143, "lcr": 0.346666666666667, "gpqa": 0.687878787878788, "omniscience": -63.05, "omniscienceAccuracy": 0.16, "omniscienceNonHallucination": 0.05892857142857144, "gdpvalNormalized": 0, "price1mInputTokens": 0.07, "price1mOutputTokens": 0.18, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 163.085868278873, "medianTimeToFirstTokenSeconds": 0.758582588999957, "isOpenWeights": true }, { "name": "MiniMax M1 40k", "shortName": "MiniMax M1 40k", "slug": "minimax-m1-40k", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "intelligenceIndex": 9.98569091349641, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0783132530120482, "critpt": null, "lcr": null, "gpqa": 0.681818181818182, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "GPT-5.4 mini (Medium)", "shortName": "GPT-5.4 mini (medium)", "slug": "gpt-5-4-mini-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 19.7484210566215, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.340909090909091, "scicode": null, "hle": 0.185820203892493, "critpt": 0.0285714285714286, "lcr": 0.67, "gpqa": 0.823232323232323, "omniscience": -20.616666666666667, "omniscienceAccuracy": 0.36466666666666664, "omniscienceNonHallucination": 0.10152151101783835, "gdpvalNormalized": null, "price1mInputTokens": 0.75, "price1mOutputTokens": 4.5, "cacheHitPrice": 0.075, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 189.939660990202, "medianTimeToFirstTokenSeconds": 8.61273873599998, "isOpenWeights": false }, { "name": "Olmo 3.1 32B Think", "shortName": "Olmo 3.1 32B Think", "slug": "olmo-3-1-32b-think", "isReasoning": true, "deprecated": false, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 7.12364412279906, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0625579240037071, "critpt": 0, "lcr": 0, "gpqa": 0.590909090909091, "omniscience": -43.36666666666667, "omniscienceAccuracy": 0.13783333333333334, "omniscienceNonHallucination": 0.3371351246858689, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65500, "isOpenWeights": true }, { "name": "Qwen3.6 27B (Non-reasoning)", "shortName": "Qwen3.6 27B (non-reasoning)", "slug": "qwen3-6-27b-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 19.8288203580036, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.51310861423221, "terminalbenchHard": 0.212121212121212, "scicode": null, "hle": 0.150602409638554, "critpt": 0.00857142857142857, "lcr": 0.666666666666667, "gpqa": 0.829292929292929, "omniscience": -53.2, "omniscienceAccuracy": 0.16716666666666666, "omniscienceNonHallucination": 0.16049629777866725, "gdpvalNormalized": 0.22494999999999998, "price1mInputTokens": 0.6, "price1mOutputTokens": 3.6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 55.1005626552233, "medianTimeToFirstTokenSeconds": 3.64927574249998, "isOpenWeights": true }, { "name": "Nex-N2-Pro (Based on Qwen3.5-397B-A17B)", "shortName": "Nex-N2-Pro", "slug": "nex-n2-pro", "isReasoning": true, "deprecated": false, "modelCreatorName": "Nex AGI", "intelligenceIndex": 28.1990972887912, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.677902621722846, "terminalbenchHard": null, "scicode": 0.43287037037037, "hle": 0.337349397590361, "critpt": 0.0857142857142857, "lcr": 0.816666666666667, "gpqa": 0.891919191919192, "omniscience": -27.05, "omniscienceAccuracy": 0.3481666666666667, "omniscienceNonHallucination": 0.050882127333162885, "gdpvalNormalized": 0.29572000000000004, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262000, "isOpenWeights": true }, { "name": "GPT-4.1", "shortName": "GPT-4.1", "slug": "gpt-4-1", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 12.6913226077818, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.136363636363636, "scicode": null, "hle": 0.0418, "critpt": 0, "lcr": 0.683333333333333, "gpqa": 0.665656565656566, "omniscience": -39.63333333333333, "omniscienceAccuracy": 0.2776666666666667, "omniscienceNonHallucination": 0.06691278264882328, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 8, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 136.234169662773, "medianTimeToFirstTokenSeconds": 0.875401955000001, "isOpenWeights": false }, { "name": "Claude Fable 5.1 (Adaptive Reasoning, Medium Effort, Default Fallback)", "shortName": "Claude Fable 5.1 (medium with fallback)", "slug": "claude-fable-5-1-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 48.9206162609523, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.982597436096585, "terminalBench40": 0.44949494949495, "terminalBench21": 0.880149812734082, "terminalbenchHard": null, "scicode": 0.563657407407407, "hle": 0.537998146431881, "critpt": 0.291428571428571, "lcr": 0.846666666666667, "gpqa": 0.885858585858586, "omniscience": 37.6, "omniscienceAccuracy": 0.631, "omniscienceNonHallucination": 0.3089430894308943, "gdpvalNormalized": 0.517925, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 0.25, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 49.5773845215713, "medianTimeToFirstTokenSeconds": 8.05103637100005, "isOpenWeights": false }, { "name": "GPT-6.1 Sol (High)", "shortName": "GPT-6.1 Sol (high)", "slug": "gpt-6-1-sol-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 50.2377777519769, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.31914442375664703, "terminalBench40": 0.515151515151515, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.55787037037037, "hle": 0.513901760889713, "critpt": 0.3, "lcr": 0.823333333333333, "gpqa": null, "omniscience": 41.45, "omniscienceAccuracy": 0.608, "omniscienceNonHallucination": 0.5063775510204082, "gdpvalNormalized": 0.49320500000000006, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.1, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 63.4706595002192, "medianTimeToFirstTokenSeconds": 57.255340688, "isOpenWeights": false }, { "name": "Command A", "shortName": "Command A", "slug": "command-a", "isReasoning": false, "deprecated": false, "modelCreatorName": "Cohere", "intelligenceIndex": 6.9581988704518, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0403151065801668, "critpt": 0, "lcr": 0.213333333333333, "gpqa": 0.527272727272727, "omniscience": -48.266666666666666, "omniscienceAccuracy": 0.16383333333333333, "omniscienceNonHallucination": 0.22682878214072155, "gdpvalNormalized": null, "price1mInputTokens": 2.5, "price1mOutputTokens": 10, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 61.6500535343768, "medianTimeToFirstTokenSeconds": 1.71899979950004, "isOpenWeights": true }, { "name": "LFM2.5-VL-1.6B", "shortName": "LFM2.5-VL-1.6B", "slug": "lfm2-5-vl-1-6b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Liquid AI", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0509731232622799, "critpt": 0, "lcr": 0, "gpqa": 0.288888888888889, "omniscience": -84.41666666666667, "omniscienceAccuracy": 0.057833333333333334, "omniscienceNonHallucination": 0.042632230673978366, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Qwen3 VL 8B Instruct", "shortName": "Qwen3 VL 8B", "slug": "qwen3-vl-8b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.25835272478202, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0268767377201112, "critpt": 0, "lcr": 0.166666666666667, "gpqa": 0.427272727272727, "omniscience": -51.88333333333333, "omniscienceAccuracy": 0.20483333333333334, "omniscienceNonHallucination": 0.08991825613079019, "gdpvalNormalized": null, "price1mInputTokens": 0.18, "price1mOutputTokens": 0.7, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 117.656957231024, "medianTimeToFirstTokenSeconds": 2.23903075250007, "isOpenWeights": true }, { "name": "JT-MINI", "shortName": "JT-MINI", "slug": "jt-mini", "isReasoning": false, "deprecated": false, "modelCreatorName": "China Mobile", "intelligenceIndex": 12.2117782837313, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.181818181818182, "scicode": null, "hle": 0.0644114921223355, "critpt": 0, "lcr": 0.13, "gpqa": 0.675757575757576, "omniscience": -64.1, "omniscienceAccuracy": 0.14216666666666666, "omniscienceNonHallucination": 0.08704099475422578, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Ling-mini-2.0", "shortName": "Ling-mini-2.0", "slug": "ling-mini-2-0", "isReasoning": false, "deprecated": true, "modelCreatorName": "InclusionAI", "intelligenceIndex": 5.54385892635499, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0505097312326228, "critpt": 0, "lcr": null, "gpqa": 0.561616161616162, "omniscience": -78.46666666666667, "omniscienceAccuracy": 0.08883333333333333, "omniscienceNonHallucination": 0.04133894274739347, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131000, "isOpenWeights": true }, { "name": "Gemini 2.0 Flash-Lite (Preview)", "shortName": "Gemini 2.0 Flash-Lite (Preview)", "slug": "gemini-2-0-flash-lite-preview", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 7.32739604924354, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0409, "critpt": null, "lcr": null, "gpqa": 0.542424242424242, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Qwen3 30B A3B (Reasoning)", "shortName": "Qwen3 30B", "slug": "qwen3-30b-a3b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.62614322282085, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0616, "critpt": 0, "lcr": 0, "gpqa": 0.616161616161616, "omniscience": -51.483333333333334, "omniscienceAccuracy": 0.16183333333333333, "omniscienceNonHallucination": 0.1926824418373434, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 2.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 107.195704508504, "medianTimeToFirstTokenSeconds": 2.22300991299997, "isOpenWeights": true }, { "name": "Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)", "shortName": "Claude Fable 5.1 (max with fallback)", "slug": "claude-fable-5-1", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 53.3549259623252, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 7.629706364004841, "terminalBench40": 0.52020202020202, "terminalBench21": 0.913857677902622, "terminalbenchHard": null, "scicode": 0.630787037037037, "hle": 0.591288229842447, "critpt": 0.297142857142857, "lcr": 0.853333333333333, "gpqa": 0.937373737373737, "omniscience": 43.45, "omniscienceAccuracy": 0.6723333333333333, "omniscienceNonHallucination": 0.2741607324516785, "gdpvalNormalized": 0.6173350000000001, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 0.25, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 69.2061618515879, "medianTimeToFirstTokenSeconds": 268.701125175, "isOpenWeights": false }, { "name": "Mixtral 8x7B Instruct", "shortName": "Mixtral 8x7B", "slug": "mixtral-8x7b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.12294610372299, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0468, "critpt": null, "lcr": null, "gpqa": 0.291919191919192, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.45, "price1mOutputTokens": 0.7, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Molmo2-8B", "shortName": "Molmo2-8B", "slug": "molmo2-8b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 5.00251209304105, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0426320667284523, "critpt": 0, "lcr": 0, "gpqa": 0.425252525252525, "omniscience": -69.21666666666667, "omniscienceAccuracy": 0.1145, "omniscienceNonHallucination": 0.08902691511387162, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 36864, "isOpenWeights": true }, { "name": "Mistral Small 4 (Reasoning)", "shortName": "Mistral Small 4", "slug": "mistral-small-4", "isReasoning": true, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 11.2675794141469, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.01504679036588765, "terminalBench40": 0, "terminalBench21": 0.209737827715356, "terminalbenchHard": 0.174242424242424, "scicode": 0.387731481481481, "hle": 0.0987025023169602, "critpt": 0.00285714285714286, "lcr": 0.496666666666667, "gpqa": 0.768686868686869, "omniscience": -30.4, "omniscienceAccuracy": 0.21683333333333332, "omniscienceNonHallucination": 0.3349648861459885, "gdpvalNormalized": 0, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.6, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 174.311241290575, "medianTimeToFirstTokenSeconds": 0.772113205500033, "isOpenWeights": true }, { "name": "Qwen3.5 0.8B (Non-reasoning)", "shortName": "Qwen3.5 0.8B (non-reasoning)", "slug": "qwen3-5-0-8b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.40008099703412, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.00374531835205993, "terminalbenchHard": 0, "scicode": null, "hle": 0.0509731232622799, "critpt": 0, "lcr": 0.08, "gpqa": 0.236363636363636, "omniscience": -88.58333333333333, "omniscienceAccuracy": 0.049833333333333334, "omniscienceNonHallucination": 0.015260480617435568, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "GLM-5 (Reasoning)", "shortName": "GLM-5", "slug": "glm-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 27.9111550817227, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.431818181818182, "scicode": null, "hle": 0.292863762743281, "critpt": 0.02, "lcr": 0.756666666666667, "gpqa": 0.82020202020202, "omniscience": 0.26666666666666666, "omniscienceAccuracy": 0.263, "omniscienceNonHallucination": 0.6467661691542288, "gdpvalNormalized": null, "price1mInputTokens": 1, "price1mOutputTokens": 3.2, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 74.1836610882773, "medianTimeToFirstTokenSeconds": 1.636406921, "isOpenWeights": true }, { "name": "Claude Opus 4.5 (Non-reasoning)", "shortName": "Claude Opus 4.5 (non-reasoning)", "slug": "claude-opus-4-5", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 23.6788975349785, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.409090909090909, "scicode": null, "hle": 0.131603336422614, "critpt": 0.00285714285714286, "lcr": 0.706666666666667, "gpqa": 0.81010101010101, "omniscience": -4.133333333333334, "omniscienceAccuracy": 0.4091666666666667, "omniscienceNonHallucination": 0.23751763046544427, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 45.3674638308523, "medianTimeToFirstTokenSeconds": 1.22973166000003, "isOpenWeights": false }, { "name": "Qwen Chat 72B", "shortName": "Qwen Chat 72B", "slug": "qwen-chat-72b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.4363632246188, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 33792, "isOpenWeights": true }, { "name": "Gemini 1.5 Flash (Sep '24)", "shortName": "Gemini 1.5 Flash (Sep)", "slug": "gemini-1-5-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 7.07110907005398, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0317, "critpt": null, "lcr": null, "gpqa": 0.462626262626263, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "GPT-5.6 Luna (Max)", "shortName": "GPT-5.6 Luna (max)", "slug": "gpt-5-6-luna", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 37.3244239690841, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.17829726152289094, "terminalBench40": 0.116161616161616, "terminalBench21": 0.808988764044944, "terminalbenchHard": null, "scicode": 0.53587962962963, "hle": 0.394810009267841, "critpt": 0.205714285714286, "lcr": 0.836666666666667, "gpqa": 0.911111111111111, "omniscience": -10.283333333333333, "omniscienceAccuracy": 0.42733333333333334, "omniscienceNonHallucination": 0.07421420256111755, "gdpvalNormalized": 0.481615, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.02, "cacheWritePrice": 0.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 112.385581258285, "medianTimeToFirstTokenSeconds": 110.9479657635, "isOpenWeights": false }, { "name": "GLM-4.6V (Non-reasoning)", "shortName": "GLM-4.6V (non-reasoning)", "slug": "glm-4-6v", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 8.35003925872647, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0303030303030303, "scicode": null, "hle": 0.0366079703429101, "critpt": 0, "lcr": 0.17, "gpqa": 0.565656565656566, "omniscience": -37.63333333333333, "omniscienceAccuracy": 0.174, "omniscienceNonHallucination": 0.33373688458434225, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 0.9, "cacheHitPrice": 0.1775, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 72.9650563948602, "medianTimeToFirstTokenSeconds": 3.54491512299998, "isOpenWeights": true }, { "name": "Solar Pro 2 (Preview) (Non-reasoning)", "shortName": "Solar Pro 2 (Preview) (non-reasoning)", "slug": "solar-pro-2-preview", "isReasoning": false, "deprecated": true, "modelCreatorName": "Upstage", "intelligenceIndex": 7.90554657851904, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0366079703429101, "critpt": null, "lcr": null, "gpqa": 0.544444444444445, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 64000, "isOpenWeights": false }, { "name": "Ling-3.0-flash-VL", "shortName": "Ling-3.0-flash-VL", "slug": "ling-3-0-flash-vl", "isReasoning": true, "deprecated": false, "modelCreatorName": "InclusionAI", "intelligenceIndex": 24.5693054077875, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.644194756554307, "terminalbenchHard": null, "scicode": 0.44212962962963, "hle": 0.219647822057461, "critpt": 0.02, "lcr": 0.783333333333333, "gpqa": 0.861616161616162, "omniscience": -4.533333333333333, "omniscienceAccuracy": 0.1435, "omniscienceNonHallucination": 0.7795290912628916, "gdpvalNormalized": 0.32494500000000004, "price1mInputTokens": 0.075, "price1mOutputTokens": 0.22, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 142.331222841403, "medianTimeToFirstTokenSeconds": 1.86435055750002, "isOpenWeights": true }, { "name": "MiniCPM-V 4.6 1.3B", "shortName": "MiniCPM-V 4.6 1.3B", "slug": "minicpm-v4-6-1-3b", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenBMB", "intelligenceIndex": 5.69026588282761, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0, "terminalbenchHard": 0, "scicode": null, "hle": 0.0505097312326228, "critpt": 0, "lcr": 0.07, "gpqa": 0.305050505050505, "omniscience": -82.86666666666666, "omniscienceAccuracy": 0.06733333333333333, "omniscienceNonHallucination": 0.03931379556826309, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262000, "isOpenWeights": true }, { "name": "Qwen3 235B A22B (Non-reasoning)", "shortName": "Qwen3 235B (non-reasoning)", "slug": "qwen3-235b-a22b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.29469219577047, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0419, "critpt": 0, "lcr": 0, "gpqa": 0.613131313131313, "omniscience": -52.38333333333333, "omniscienceAccuracy": 0.18566666666666667, "omniscienceNonHallucination": 0.12873516168645105, "gdpvalNormalized": null, "price1mInputTokens": 0.7, "price1mOutputTokens": 2.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 58.6621008803587, "medianTimeToFirstTokenSeconds": 2.72342781750003, "isOpenWeights": true }, { "name": "Claude Opus 4.6 (Non-reasoning, High Effort)", "shortName": "Claude Opus 4.6 (non-reasoning, high)", "slug": "claude-opus-4-6", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 26.3534088054787, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.484848484848485, "scicode": null, "hle": 0.190917516218721, "critpt": 0.0284081632653061, "lcr": 0.67, "gpqa": 0.84040404040404, "omniscience": 2.3666666666666667, "omniscienceAccuracy": 0.4578333333333333, "omniscienceNonHallucination": 0.19920073778051028, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 37.1903214591269, "medianTimeToFirstTokenSeconds": 2.43486785749998, "isOpenWeights": false }, { "name": "Mistral Small 3.2", "shortName": "Mistral Small 3.2", "slug": "mistral-small-3-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 8.24541426908029, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.0561797752808989, "terminalbenchHard": 0.0681818181818182, "scicode": 0.28587962962963, "hle": 0.0430954587581094, "critpt": 0, "lcr": 0.203333333333333, "gpqa": 0.505050505050505, "omniscience": -50.583333333333336, "omniscienceAccuracy": 0.14833333333333334, "omniscienceNonHallucination": 0.23189823874755378, "gdpvalNormalized": 0, "price1mInputTokens": 0.075, "price1mOutputTokens": 0.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "DeepSeek V3 (Dec '24)", "shortName": "DeepSeek V3 (Dec)", "slug": "deepseek-v3", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 8.48686760841828, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.019751793975774472, "terminalBench40": 0, "terminalBench21": 0.168539325842697, "terminalbenchHard": 0.0681818181818182, "scicode": 0.357638888888889, "hle": 0.029, "critpt": 0, "lcr": 0.293333333333333, "gpqa": 0.556565656565657, "omniscience": -41.65, "omniscienceAccuracy": 0.2545, "omniscienceNonHallucination": 0.09993293091884636, "gdpvalNormalized": 0, "price1mInputTokens": 0.32, "price1mOutputTokens": 0.89, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Claude Opus 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)", "shortName": "Claude Opus 5.5 (low with fallback)", "slug": "claude-opus-5-5-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 42.3077781466168, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.551180473909146, "terminalBench40": 0.313131313131313, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.585648148148148, "hle": 0.483317886932345, "critpt": 0.177142857142857, "lcr": 0.806666666666667, "gpqa": null, "omniscience": 38.86666666666667, "omniscienceAccuracy": 0.6353333333333333, "omniscienceNonHallucination": 0.32358318098720296, "gdpvalNormalized": 0.361755, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.2, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 72.1873128039508, "medianTimeToFirstTokenSeconds": 12.8949449205, "isOpenWeights": false }, { "name": "Kimi K2", "shortName": "Kimi K2", "slug": "kimi-k2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 12.7154830817268, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.159090909090909, "scicode": null, "hle": 0.0741427247451344, "critpt": 0, "lcr": 0.53, "gpqa": 0.765656565656566, "omniscience": -28.316666666666666, "omniscienceAccuracy": 0.2735, "omniscienceNonHallucination": 0.23376921312227572, "gdpvalNormalized": null, "price1mInputTokens": 0.57, "price1mOutputTokens": 2.3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 52.5450114081741, "medianTimeToFirstTokenSeconds": 1.53609237300001, "isOpenWeights": true }, { "name": "DeepSeek R1 Distill Qwen 1.5B", "shortName": "DeepSeek R1 Distill Qwen 1.5B", "slug": "deepseek-r1-distill-qwen-1-5b", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 5.51106155192649, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0308, "critpt": null, "lcr": 0.0166666666666667, "gpqa": 0.097979797979798, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "EXAONE 4.0 32B (Reasoning)", "shortName": "EXAONE 4.0 32B", "slug": "exaone-4-0-32b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "LG AI Research", "intelligenceIndex": 8.17836973273723, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.114457831325301, "critpt": 0, "lcr": null, "gpqa": 0.739393939393939, "omniscience": -60.06666666666667, "omniscienceAccuracy": 0.13866666666666666, "omniscienceNonHallucination": 0.14164086687306499, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131000, "isOpenWeights": true }, { "name": "Solar Pro 2 (Non-reasoning)", "shortName": "Solar Pro 2 (non-reasoning)", "slug": "solar-pro-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Upstage", "intelligenceIndex": 6.99835187746333, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0370713623725672, "critpt": 0, "lcr": 0, "gpqa": 0.560606060606061, "omniscience": -62.05, "omniscienceAccuracy": 0.1605, "omniscienceNonHallucination": 0.06968433591423462, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": false }, { "name": "Qwen3 14B (Reasoning)", "shortName": "Qwen3 14B", "slug": "qwen3-14b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.15433529883621, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.048689138576779, "terminalbenchHard": 0.0378787878787879, "scicode": 0.306712962962963, "hle": 0.0453, "critpt": 0, "lcr": 0, "gpqa": 0.604040404040404, "omniscience": -48.95, "omniscienceAccuracy": 0.15283333333333332, "omniscienceNonHallucination": 0.24178634664568166, "gdpvalNormalized": 0, "price1mInputTokens": 0.35, "price1mOutputTokens": 4.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 61.5372970347416, "medianTimeToFirstTokenSeconds": 2.74222165750006, "isOpenWeights": true }, { "name": "Sarvam 105B (High)", "shortName": "Sarvam 105B (high)", "slug": "sarvam-105b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Sarvam", "intelligenceIndex": 8.7905508357489, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.110287303058387, "critpt": 0, "lcr": 0, "gpqa": 0.738383838383838, "omniscience": -59.38333333333333, "omniscienceAccuracy": 0.176, "omniscienceNonHallucination": 0.06573624595469252, "gdpvalNormalized": null, "price1mInputTokens": 0.042, "price1mOutputTokens": 0.17, "cacheHitPrice": 0.026, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Llama 3.1 Instruct 405B", "shortName": "Llama 3.1 405B", "slug": "llama-3-1-instruct-405b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 7.2893092121128, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0398, "critpt": 0, "lcr": 0.253333333333333, "gpqa": 0.515151515151515, "omniscience": -17.1, "omniscienceAccuracy": 0.23183333333333334, "omniscienceNonHallucination": 0.4755912345411152, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Llama 3.1 Instruct 70B", "shortName": "Llama 3.1 70B", "slug": "llama-3-1-instruct-70b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 6.60404957812628, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0303030303030303, "scicode": null, "hle": 0.0447, "critpt": 0, "lcr": null, "gpqa": 0.409090909090909, "omniscience": -43.1, "omniscienceAccuracy": 0.197, "omniscienceNonHallucination": 0.2179327521793275, "gdpvalNormalized": null, "price1mInputTokens": 0.56, "price1mOutputTokens": 0.56, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 56.3969811989799, "medianTimeToFirstTokenSeconds": 1.76332569200008, "isOpenWeights": true }, { "name": "Qwen3.5 35B A3B (Reasoning)", "shortName": "Qwen3.5 35B A3B", "slug": "qwen3-5-35b-a3b", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 19.3295292724173, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.265151515151515, "scicode": null, "hle": 0.210379981464319, "critpt": 0.00857142857142857, "lcr": 0.72, "gpqa": 0.845454545454545, "omniscience": -48.1, "omniscienceAccuracy": 0.20133333333333334, "omniscienceNonHallucination": 0.14565943238731216, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 149.345816170568, "medianTimeToFirstTokenSeconds": 2.10864810649997, "isOpenWeights": true }, { "name": "JT-4.1 Flash 236B A21B", "shortName": "JT-4.1 Flash 236B A21B", "slug": "jt236b", "isReasoning": true, "deprecated": false, "modelCreatorName": "China Mobile", "intelligenceIndex": 33.9172581709394, "intelligenceIndexIsEstimated": false, "terminalBench40": 0.0303030303030303, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.519675925925926, "hle": 0.435588507877665, "critpt": 0.14, "lcr": 0.783333333333333, "gpqa": null, "omniscience": 20.566666666666666, "omniscienceAccuracy": 0.494, "omniscienceNonHallucination": 0.430171277997365, "gdpvalNormalized": 0.40804499999999994, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": false }, { "name": "MiMo-V2.5-Pro (Non-reasoning)", "shortName": "MiMo-V2.5-Pro (non-reasoning)", "slug": "mimo-v2-5-pro-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 18.2900328799708, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.356060606060606, "scicode": null, "hle": 0.147822057460612, "critpt": 0.0114285714285714, "lcr": 0.416666666666667, "gpqa": 0.761616161616162, "omniscience": -37.8, "omniscienceAccuracy": 0.2718333333333333, "omniscienceNonHallucination": 0.10757610437170972, "gdpvalNormalized": null, "price1mInputTokens": 0.435, "price1mOutputTokens": 0.87, "cacheHitPrice": 0.0036, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 23.2224391294025, "medianTimeToFirstTokenSeconds": 2.1806715175, "isOpenWeights": true }, { "name": "Ring-flash-2.0", "shortName": "Ring-flash-2.0", "slug": "ring-flash-2-0", "isReasoning": true, "deprecated": false, "modelCreatorName": "InclusionAI", "intelligenceIndex": 7.15340447175535, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0757575757575758, "scicode": null, "hle": 0.0963855421686747, "critpt": 0.00277551020408163, "lcr": 0.21, "gpqa": 0.725252525252525, "omniscience": -58.61666666666667, "omniscienceAccuracy": 0.1655, "omniscienceNonHallucination": 0.09926103455162771, "gdpvalNormalized": null, "price1mInputTokens": 0.14, "price1mOutputTokens": 0.57, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "LFM2.5-1.2B-Instruct", "shortName": "LFM2.5-1.2B-Instruct", "slug": "lfm2-5-1-2b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Liquid AI", "intelligenceIndex": 5.20951181661378, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.067191844300278, "critpt": 0, "lcr": 0, "gpqa": 0.326262626262626, "omniscience": -72.1, "omniscienceAccuracy": 0.07016666666666667, "omniscienceNonHallucination": 0.14913066857859836, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "GPT-5 (High)", "shortName": "GPT-5 (high)", "slug": "gpt-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 22.9827995216189, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.352059925093633, "terminalbenchHard": 0.325757575757576, "scicode": null, "hle": 0.28498609823911, "critpt": 0.0571428571428571, "lcr": 0.782, "gpqa": 0.853535353535353, "omniscience": -8.733333333333333, "omniscienceAccuracy": 0.4031666666666667, "omniscienceNonHallucination": 0.17816252443451552, "gdpvalNormalized": 0.20288, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 76.9756362138456, "medianTimeToFirstTokenSeconds": 75.4556100575, "isOpenWeights": false }, { "name": "Mistral Large 3", "shortName": "Mistral Large 3", "slug": "mistral-large-3", "isReasoning": false, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 9.26608812860467, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.031125992260624492, "terminalBench40": 0, "terminalBench21": 0.119850187265918, "terminalbenchHard": 0.159090909090909, "scicode": 0.365740740740741, "hle": 0.0417052826691381, "critpt": 0, "lcr": 0.36, "gpqa": 0.67979797979798, "omniscience": -39.56666666666667, "omniscienceAccuracy": 0.2495, "omniscienceNonHallucination": 0.14035087719298245, "gdpvalNormalized": 0, "price1mInputTokens": 0.5, "price1mOutputTokens": 1.5, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 78.5757063816355, "medianTimeToFirstTokenSeconds": 1.07980729149998, "isOpenWeights": true }, { "name": "GPT-5.4 (Non-reasoning)", "shortName": "GPT-5.4 (non-reasoning)", "slug": "gpt-5-4-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 18.1597022799287, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.378787878787879, "scicode": null, "hle": 0.112604263206673, "critpt": 0.00571428571428571, "lcr": 0.583333333333333, "gpqa": 0.748484848484848, "omniscience": -15.666666666666666, "omniscienceAccuracy": 0.37383333333333335, "omniscienceNonHallucination": 0.1527814745807825, "gdpvalNormalized": null, "price1mInputTokens": 2.5, "price1mOutputTokens": 15, "cacheHitPrice": 0.25, "cacheWritePrice": null, "contextWindowTokens": 1050000, "medianOutputTokensPerSecond": 66.4169470734175, "medianTimeToFirstTokenSeconds": 0.883259496000008, "isOpenWeights": false }, { "name": "GPT-5.2 (Xhigh)", "shortName": "GPT-5.2 (xhigh)", "slug": "gpt-5-2", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 30.448181916806, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.46969696969697, "scicode": null, "hle": 0.376737720111214, "critpt": 0.115510204081633, "lcr": 0.826666666666667, "gpqa": 0.903030303030303, "omniscience": -0.8833333333333333, "omniscienceAccuracy": 0.44316666666666665, "omniscienceNonHallucination": 0.18826698593235558, "gdpvalNormalized": null, "price1mInputTokens": 1.75, "price1mOutputTokens": 14, "cacheHitPrice": 0.175, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 67.9111012359726, "medianTimeToFirstTokenSeconds": 132.48270213, "isOpenWeights": false }, { "name": "Apertus 70B Instruct", "shortName": "Apertus 70B Instruct", "slug": "apertus-70b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Swiss AI Initiative", "intelligenceIndex": 5.11462775284761, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0546802594995366, "critpt": 0, "lcr": 0, "gpqa": 0.271717171717172, "omniscience": -61.55, "omniscienceAccuracy": 0.11633333333333333, "omniscienceNonHallucination": 0.1718219539796303, "gdpvalNormalized": null, "price1mInputTokens": 0.82, "price1mOutputTokens": 2.92, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "Grok 4.1 Fast (Non-reasoning)", "shortName": "Grok 4.1 Fast (non-reasoning)", "slug": "grok-4-1-fast", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 11.2849071635465, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.143939393939394, "scicode": null, "hle": 0.0509731232622799, "critpt": 0, "lcr": 0.313333333333333, "gpqa": 0.637373737373737, "omniscience": -50.88333333333333, "omniscienceAccuracy": 0.17233333333333334, "omniscienceNonHallucination": 0.17700362464760366, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "DeepSeek R1 Distill Llama 70B", "shortName": "DeepSeek R1 Distill Llama 70B", "slug": "deepseek-r1-distill-llama-70b", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 7.88824581191087, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0513, "critpt": 0, "lcr": 0.1, "gpqa": 0.402020202020202, "omniscience": -46.38333333333333, "omniscienceAccuracy": 0.19166666666666668, "omniscienceNonHallucination": 0.18907216494845358, "gdpvalNormalized": null, "price1mInputTokens": 0.7, "price1mOutputTokens": 1.1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3 0.6B (Reasoning)", "shortName": "Qwen3 0.6B", "slug": "qwen3-0.6b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0564, "critpt": 0, "lcr": 0, "gpqa": 0.239393939393939, "omniscience": -78.85, "omniscienceAccuracy": 0.06333333333333334, "omniscienceNonHallucination": 0.0905693950177936, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "MiniMax-M2.7", "shortName": "MiniMax-M2.7", "slug": "minimax-m2-7", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "intelligenceIndex": 22.7578150287271, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.554307116104869, "terminalbenchHard": 0.393939393939394, "scicode": 0.501157407407407, "hle": 0.29610750695088, "critpt": 0.00571428571428571, "lcr": 0.783333333333333, "gpqa": 0.873737373737374, "omniscience": 0.7666666666666667, "omniscienceAccuracy": 0.268, "omniscienceNonHallucination": 0.6443533697632058, "gdpvalNormalized": 0.24931999999999999, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.06, "cacheWritePrice": 0.375, "contextWindowTokens": 204800, "medianOutputTokensPerSecond": 48.7892394715947, "medianTimeToFirstTokenSeconds": 1.618382475, "isOpenWeights": true }, { "name": "Grok-1", "shortName": "Grok-1", "slug": "grok-1", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 6.33793271707089, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "Doubao Seed Code", "shortName": "Doubao Seed Code", "slug": "doubao-seed-code", "isReasoning": true, "deprecated": false, "modelCreatorName": "ByteDance Seed", "intelligenceIndex": 16.9467715940932, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.265151515151515, "scicode": null, "hle": 0.140871177015755, "critpt": 0.00285714285714286, "lcr": 0.716666666666667, "gpqa": 0.763636363636364, "omniscience": -35.2, "omniscienceAccuracy": 0.24883333333333332, "omniscienceNonHallucination": 0.20013312624805857, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "GPT-5.1 (High)", "shortName": "GPT-5.1 (high)", "slug": "gpt-5-1", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 24.7357540558308, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.524344569288389, "terminalbenchHard": 0.454545454545455, "scicode": null, "hle": 0.28498609823911, "critpt": 0.0485714285714286, "lcr": 0.8, "gpqa": 0.872727272727273, "omniscience": 5.4, "omniscienceAccuracy": 0.37716666666666665, "omniscienceNonHallucination": 0.4811345999464811, "gdpvalNormalized": 0.15631, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 272000, "medianOutputTokensPerSecond": 101.323830514383, "medianTimeToFirstTokenSeconds": 38.3983523685, "isOpenWeights": false }, { "name": "Gemini 3.5 Flash (Medium)", "shortName": "Gemini 3.5 Flash (medium)", "slug": "gemini-3-5-flash-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 33.6325871590289, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.393939393939394, "scicode": null, "hle": 0.413345690454124, "critpt": 0.108571428571429, "lcr": 0.743333333333333, "gpqa": 0.921212121212121, "omniscience": 20.816666666666666, "omniscienceAccuracy": 0.5105, "omniscienceNonHallucination": 0.3823629553966632, "gdpvalNormalized": null, "price1mInputTokens": 1.5, "price1mOutputTokens": 9, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 197.602280254338, "medianTimeToFirstTokenSeconds": 14.92168615, "isOpenWeights": false }, { "name": "Claude 4 Opus (Non-reasoning)", "shortName": "Claude 4 Opus (non-reasoning)", "slug": "claude-4-opus", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 16.6142232043434, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.061631139944393, "critpt": null, "lcr": null, "gpqa": 0.701010101010101, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 15, "price1mOutputTokens": 75, "cacheHitPrice": 1.5, "cacheWritePrice": 18.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "EXAONE 4.5 33B", "shortName": "EXAONE 4.5 33B", "slug": "exaone-4-5-33b", "isReasoning": true, "deprecated": false, "modelCreatorName": "LG AI Research", "intelligenceIndex": 13.2058394009382, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.213483146067416, "terminalbenchHard": 0.204545454545455, "scicode": null, "hle": 0.128822984244671, "critpt": 0.00285714285714286, "lcr": 0.546666666666667, "gpqa": 0.793939393939394, "omniscience": -50.71666666666667, "omniscienceAccuracy": 0.16233333333333333, "omniscienceNonHallucination": 0.20075606844409077, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Qwen3.5 27B (Non-reasoning)", "shortName": "Qwen3.5 27B (non-reasoning)", "slug": "qwen3-5-27b-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 19.3684268009412, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.318181818181818, "scicode": null, "hle": 0.139481000926784, "critpt": 0.00285714285714286, "lcr": 0.64, "gpqa": 0.842424242424242, "omniscience": -47.666666666666664, "omniscienceAccuracy": 0.15683333333333332, "omniscienceNonHallucination": 0.24866574421822496, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 83.1385005669007, "medianTimeToFirstTokenSeconds": 5.68086882499989, "isOpenWeights": true }, { "name": "Mistral Large 2 (Nov '24)", "shortName": "Mistral Large 2 (Nov)", "slug": "mistral-large-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 7.55559683895193, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0334, "critpt": 0, "lcr": null, "gpqa": 0.485858585858586, "omniscience": -34.36666666666667, "omniscienceAccuracy": 0.19866666666666666, "omniscienceNonHallucination": 0.3232113144758736, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Kimi K3 (Low)", "shortName": "Kimi K3 (low)", "slug": "kimi-k3-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Kimi", "intelligenceIndex": 30.0666987529068, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.1463056165227514, "terminalBench40": 0.126262626262626, "terminalBench21": 0.823970037453184, "terminalbenchHard": null, "scicode": 0.52662037037037, "hle": 0.249768303985171, "critpt": 0.0314285714285714, "lcr": 0.793333333333333, "gpqa": 0.842424242424242, "omniscience": 3.9166666666666665, "omniscienceAccuracy": 0.4575, "omniscienceNonHallucination": 0.22887864823348691, "gdpvalNormalized": 0.30929999999999996, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": null, "contextWindowTokens": 1048576, "medianOutputTokensPerSecond": 36.091326367411, "medianTimeToFirstTokenSeconds": 5.58204729, "isOpenWeights": true }, { "name": "PALM-2", "shortName": "PALM-2", "slug": "palm-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 5.36921134713752, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8000, "isOpenWeights": false }, { "name": "Qwen3 VL 30B A3B Instruct", "shortName": "Qwen3 VL 30B A3B", "slug": "qwen3-vl-30b-a3b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.9263607036835, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0630213160333642, "critpt": 0, "lcr": null, "gpqa": 0.694949494949495, "omniscience": -63.416666666666664, "omniscienceAccuracy": 0.15616666666666668, "omniscienceNonHallucination": 0.06340114556587007, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 113.805447635537, "medianTimeToFirstTokenSeconds": 2.171983915, "isOpenWeights": true }, { "name": "Grok 4.20 0309 (Non-reasoning)", "shortName": "Grok 4.20 0309 (non-reasoning)", "slug": "grok-4-20-0309-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 14.606641173969, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.21969696969697, "scicode": null, "hle": 0.245134383688601, "critpt": 0, "lcr": 0.253333333333333, "gpqa": 0.784848484848485, "omniscience": -42.583333333333336, "omniscienceAccuracy": 0.2605, "omniscienceNonHallucination": 0.07189542483660127, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "DeepSeek V3.2 Speciale", "shortName": "DeepSeek V3.2 Speciale", "slug": "deepseek-v3-2-speciale", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 14.4560679389612, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.348484848484849, "scicode": null, "hle": 0.287303058387396, "critpt": 0.0742857142857143, "lcr": 0.7, "gpqa": 0.870707070707071, "omniscience": -17.716666666666665, "omniscienceAccuracy": 0.37766666666666665, "omniscienceNonHallucination": 0.10846277450455277, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-6 Sol (High)", "shortName": "GPT-6 Sol (high)", "slug": "gpt-6-sol-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 42.8215513642985, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.3753396394543572, "terminalBench40": 0.262626262626263, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.548611111111111, "hle": 0.44114921223355, "critpt": 0.254285714285714, "lcr": 0.836666666666667, "gpqa": null, "omniscience": 26.816666666666666, "omniscienceAccuracy": 0.5371666666666667, "omniscienceNonHallucination": 0.4187972632337055, "gdpvalNormalized": 0.438025, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 872000, "medianOutputTokensPerSecond": 63.4240248606367, "medianTimeToFirstTokenSeconds": 19.1963296525, "isOpenWeights": false }, { "name": "Gemma 4 E4B (Non-reasoning)", "shortName": "Gemma 4 E4B (non-reasoning)", "slug": "gemma-4-e4b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 7.45765296912625, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0757575757575758, "scicode": null, "hle": 0.0481927710843374, "critpt": 0.00285714285714286, "lcr": 0.24, "gpqa": 0.549494949494949, "omniscience": -40.983333333333334, "omniscienceAccuracy": 0.08316666666666667, "omniscienceNonHallucination": 0.46227958552990367, "gdpvalNormalized": null, "price1mInputTokens": 0.02, "price1mOutputTokens": 0.1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 72.1022295062802, "medianTimeToFirstTokenSeconds": 0.794963091500079, "isOpenWeights": true }, { "name": "GPT-5.4 (Low)", "shortName": "GPT-5.4 (low)", "slug": "gpt-5-4-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 27.5773176863368, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.431818181818182, "scicode": null, "hle": 0.308155699721965, "critpt": 0.0742857142857143, "lcr": 0.766666666666667, "gpqa": 0.870707070707071, "omniscience": 4.783333333333333, "omniscienceAccuracy": 0.4785, "omniscienceNonHallucination": 0.1741770533716842, "gdpvalNormalized": null, "price1mInputTokens": 2.5, "price1mOutputTokens": 15, "cacheHitPrice": 0.25, "cacheWritePrice": null, "contextWindowTokens": 1050000, "medianOutputTokensPerSecond": 61.4466552166471, "medianTimeToFirstTokenSeconds": 2.03148759499999, "isOpenWeights": false }, { "name": "K2 Horizon MoVA 36B A4B", "shortName": "K2 Horizon MoVA 36B A4B", "slug": "k2-horizon-mova-36b-a4b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 25.3037790731435, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.565543071161049, "terminalbenchHard": null, "scicode": 0.400462962962963, "hle": 0.234476367006487, "critpt": 0.0228571428571429, "lcr": 0.716666666666667, "gpqa": 0.822222222222222, "omniscience": -10.95, "omniscienceAccuracy": 0.15, "omniscienceNonHallucination": 0.6947058823529412, "gdpvalNormalized": 0.34064, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 524288, "isOpenWeights": true }, { "name": "Claude Opus 4.6 (Adaptive Reasoning, Max Effort)", "shortName": "Claude Opus 4.6 (max)", "slug": "claude-opus-4-6-adaptive", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 31.9457397504366, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.462121212121212, "scicode": null, "hle": 0.399443929564411, "critpt": 0.125714285714286, "lcr": 0.78, "gpqa": 0.895959595959596, "omniscience": 13.666666666666666, "omniscienceAccuracy": 0.4698333333333333, "omniscienceNonHallucination": 0.3715812637535366, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 38.1281960000096, "medianTimeToFirstTokenSeconds": 16.4519737250001, "isOpenWeights": false }, { "name": "Jamba 1.5 Mini", "shortName": "Jamba 1.5 Mini", "slug": "jamba-1-5-mini", "isReasoning": false, "deprecated": true, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 5.20641262573048, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0514, "critpt": null, "lcr": null, "gpqa": 0.302020202020202, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "OLMo 2 32B", "shortName": "OLMo 2 32B", "slug": "olmo-2-32b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 5.9708764479062, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.036144578313253, "critpt": null, "lcr": 0, "gpqa": 0.328282828282828, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "Nova Pro", "shortName": "Nova Pro", "slug": "nova-pro", "isReasoning": false, "deprecated": true, "modelCreatorName": "Amazon", "intelligenceIndex": 6.95832302287488, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0319, "critpt": 0, "lcr": 0.21, "gpqa": 0.498989898989899, "omniscience": -47.733333333333334, "omniscienceAccuracy": 0.16866666666666666, "omniscienceNonHallucination": 0.22293504410585407, "gdpvalNormalized": null, "price1mInputTokens": 0.8, "price1mOutputTokens": 3.2, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 300000, "isOpenWeights": false }, { "name": "Granite 4.0 350M", "shortName": "Granite 4.0 350M", "slug": "granite-4-0-350m", "isReasoning": false, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0551436515291937, "critpt": 0, "lcr": 0, "gpqa": 0.260606060606061, "omniscience": -69.35, "omniscienceAccuracy": 0.0385, "omniscienceNonHallucination": 0.2386895475819033, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Gemini 2.5 Flash Preview (Non-reasoning)", "shortName": "Gemini 2.5 Flash (Apr) (non-reasoning)", "slug": "gemini-2-5-flash-04-2025", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 8.65865427868212, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.038, "critpt": null, "lcr": null, "gpqa": 0.593939393939394, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Grok 4 Fast (Reasoning)", "shortName": "Grok 4 Fast", "slug": "grok-4-fast-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 17.9399302586701, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.189393939393939, "scicode": null, "hle": 0.190917516218721, "critpt": 0.0285714285714286, "lcr": 0.736666666666667, "gpqa": 0.847474747474747, "omniscience": -29.933333333333334, "omniscienceAccuracy": 0.22783333333333333, "omniscienceNonHallucination": 0.3172890135981006, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "GPT-4o (March 2025, chatgpt-4o-latest)", "shortName": "GPT-4o (Mar)", "slug": "gpt-4o-chatgpt-03-25", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 8.96179172361777, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0397, "critpt": null, "lcr": null, "gpqa": 0.654545454545455, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Muse Glimmer (High)", "shortName": "Muse Glimmer (high)", "slug": "muse-glimmer", "isReasoning": true, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 17.4754176089465, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.05670634665029995, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.51685393258427, "terminalbenchHard": null, "scicode": 0.449074074074074, "hle": 0.219647822057461, "critpt": 0.0257142857142857, "lcr": 0.833333333333333, "gpqa": 0.835353535353535, "omniscience": -32.85, "omniscienceAccuracy": 0.2698333333333333, "omniscienceNonHallucination": 0.18055238530015982, "gdpvalNormalized": 0.13712, "price1mInputTokens": 0.32499999999999996, "price1mOutputTokens": 1.35, "cacheHitPrice": 0.04, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 148.099621197697, "medianTimeToFirstTokenSeconds": 1.42090929049999, "isOpenWeights": true }, { "name": "GPT-5.6 Luna (Medium)", "shortName": "GPT-5.6 Luna (medium)", "slug": "gpt-5-6-luna-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 25.0358741440784, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.015598049609820011, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.531835205992509, "terminalbenchHard": null, "scicode": 0.467592592592593, "hle": 0.257645968489342, "critpt": 0.0485714285714286, "lcr": 0.75, "gpqa": 0.858585858585859, "omniscience": -13.183333333333334, "omniscienceAccuracy": 0.407, "omniscienceNonHallucination": 0.09134345137717814, "gdpvalNormalized": 0.309095, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.02, "cacheWritePrice": 0.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 109.346776363957, "medianTimeToFirstTokenSeconds": 2.70352744950001, "isOpenWeights": false }, { "name": "Grok 4.7 (Xhigh)", "shortName": "Grok 4.7 (xhigh)", "slug": "grok-4-7", "isReasoning": true, "deprecated": false, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 46.4465506302286, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 3.738325952106939, "terminalBench40": 0.257575757575758, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.574074074074074, "hle": 0.431417979610751, "critpt": 0.177142857142857, "lcr": 0.766666666666667, "gpqa": null, "omniscience": 32.03333333333333, "omniscienceAccuracy": 0.4745, "omniscienceNonHallucination": 0.70662860767523, "gdpvalNormalized": 0.597605, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 72.8420975640514, "medianTimeToFirstTokenSeconds": 78.693411536, "isOpenWeights": false }, { "name": "Qwen3 1.7B (Reasoning)", "shortName": "Qwen3 1.7B", "slug": "qwen3-1.7b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.1865756338608, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0464, "critpt": 0, "lcr": 0, "gpqa": 0.355555555555556, "omniscience": -77.53333333333333, "omniscienceAccuracy": 0.08866666666666667, "omniscienceNonHallucination": 0.05193855157278715, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Gemini 3.1 Flash-Lite", "shortName": "Gemini 3.1 Flash-Lite", "slug": "gemini-3-1-flash-lite-preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 15.554288713025, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.03940671951220252, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.310861423220974, "terminalbenchHard": 0.242424242424242, "scicode": 0.434027777777778, "hle": 0.17191844300278, "critpt": 0.0114285714285714, "lcr": 0.743333333333333, "gpqa": 0.822222222222222, "omniscience": -16.416666666666668, "omniscienceAccuracy": 0.3626666666666667, "omniscienceNonHallucination": 0.17337866108786615, "gdpvalNormalized": 0, "price1mInputTokens": 0.25, "price1mOutputTokens": 1.5, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 249.549610901774, "medianTimeToFirstTokenSeconds": 5.385601361, "isOpenWeights": false }, { "name": "Kimi Linear 48B A3B Instruct", "shortName": "Kimi Linear 48B A3B Instruct", "slug": "kimi-linear-48b-a3b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Kimi", "intelligenceIndex": 7.30037362771236, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.113636363636364, "scicode": null, "hle": 0.0245597775718258, "critpt": null, "lcr": 0.28, "gpqa": 0.412121212121212, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "Llama 2 Chat 70B", "shortName": "Llama 2 Chat 70B", "slug": "llama-2-chat-70b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 5.3045047203477, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0521, "critpt": null, "lcr": null, "gpqa": 0.327272727272727, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "Solar Open 100B (Reasoning)", "shortName": "Solar Open 100B", "slug": "solar-open-100b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Upstage", "intelligenceIndex": 10.3644352600556, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.104263206672845, "critpt": 0, "lcr": 0.41, "gpqa": 0.656565656565657, "omniscience": -54.5, "omniscienceAccuracy": 0.18533333333333332, "omniscienceNonHallucination": 0.103518821603928, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-5.3 Codex (Xhigh)", "shortName": "GPT-5.3 Codex (xhigh)", "slug": "gpt-5-3-codex", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 32.5028174368983, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.53030303030303, "scicode": null, "hle": 0.424930491195551, "critpt": 0.168571428571429, "lcr": 0.833333333333333, "gpqa": 0.915151515151515, "omniscience": 10.866666666666667, "omniscienceAccuracy": 0.5288333333333334, "omniscienceNonHallucination": 0.1082419525999293, "gdpvalNormalized": null, "price1mInputTokens": 1.75, "price1mOutputTokens": 14, "cacheHitPrice": 0.175, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 84.9644290862387, "medianTimeToFirstTokenSeconds": 75.6801920564999, "isOpenWeights": false }, { "name": "Mistral Medium 3", "shortName": "Mistral Medium 3", "slug": "mistral-medium-3", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 9.04757819034062, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0405, "critpt": 0, "lcr": 0.313333333333333, "gpqa": 0.577777777777778, "omniscience": -31.4, "omniscienceAccuracy": 0.18333333333333332, "omniscienceNonHallucination": 0.39102040816326533, "gdpvalNormalized": null, "price1mInputTokens": 0.4, "price1mOutputTokens": 2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "LFM2.5-1.2B-Thinking", "shortName": "LFM2.5-1.2B-Thinking", "slug": "lfm2-5-1-2b-thinking", "isReasoning": true, "deprecated": false, "modelCreatorName": "Liquid AI", "intelligenceIndex": 5.22188553175398, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.06209453197405, "critpt": 0, "lcr": 0, "gpqa": 0.339393939393939, "omniscience": -79.95, "omniscienceAccuracy": 0.07416666666666667, "omniscienceNonHallucination": 0.056345634563456315, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "K2 Think V2", "shortName": "K2 Think V2", "slug": "k2-think-v2", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 11.4956908518183, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.149812734082397, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.101019462465246, "critpt": 0, "lcr": 0.57, "gpqa": 0.713131313131313, "omniscience": -40.333333333333336, "omniscienceAccuracy": 0.18016666666666667, "omniscienceNonHallucination": 0.28826997357186424, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "GLM-5.1 (Reasoning)", "shortName": "GLM-5.1", "slug": "glm-5-1", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 26.0585912980095, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.0192564984972659, "terminalBench40": 0.0202020202020202, "terminalBench21": 0.617977528089888, "terminalbenchHard": 0.431818181818182, "scicode": 0.447916666666667, "hle": 0.300741427247451, "critpt": 0.0457142857142857, "lcr": 0.736666666666667, "gpqa": 0.867676767676768, "omniscience": 0.85, "omniscienceAccuracy": 0.237, "omniscienceNonHallucination": 0.700524246395806, "gdpvalNormalized": 0.30167999999999995, "price1mInputTokens": 1.285, "price1mOutputTokens": 4.07, "cacheHitPrice": 0.26, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 44.4518797071224, "medianTimeToFirstTokenSeconds": 1.73581942099997, "isOpenWeights": true }, { "name": "o1", "shortName": "o1", "slug": "o1", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 15.2309422339403, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.128787878787879, "scicode": null, "hle": 0.0701, "critpt": 0.00285714285714286, "lcr": 0.65, "gpqa": 0.747474747474747, "omniscience": -11.05, "omniscienceAccuracy": 0.3453333333333333, "omniscienceNonHallucination": 0.3037169042769857, "gdpvalNormalized": null, "price1mInputTokens": 15, "price1mOutputTokens": 60, "cacheHitPrice": 7.5, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Qwen3 Max (Preview)", "shortName": "Qwen3 Max (Preview)", "slug": "qwen3-max-preview", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 12.5883955174617, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.196969696969697, "scicode": null, "hle": 0.101482854494903, "critpt": 0.00857142857142857, "lcr": 0.43, "gpqa": 0.763636363636364, "omniscience": -42.78333333333333, "omniscienceAccuracy": 0.242, "omniscienceNonHallucination": 0.11631486367634125, "gdpvalNormalized": null, "price1mInputTokens": 1.2, "price1mOutputTokens": 6, "cacheHitPrice": 0.24, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 55.9457437911555, "medianTimeToFirstTokenSeconds": 4.04558092050001, "isOpenWeights": false }, { "name": "Nemotron 3 Ultra 550B A55B (Reasoning)", "shortName": "Nemotron 3 Ultra", "slug": "nvidia-nemotron-3-ultra-550b-a55b", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 22.9279123422858, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5975984799533358, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.539325842696629, "terminalbenchHard": 0.363636363636364, "scicode": 0.402777777777778, "hle": 0.284059314179796, "critpt": 0.0314285714285714, "lcr": 0.793333333333333, "gpqa": 0.866666666666667, "omniscience": -0.4, "omniscienceAccuracy": 0.22583333333333333, "omniscienceNonHallucination": 0.7031216361679224, "gdpvalNormalized": 0.25017500000000004, "price1mInputTokens": 0.6, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.16, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 179.841133366299, "medianTimeToFirstTokenSeconds": 1.223322118, "isOpenWeights": true }, { "name": "LongCat 2.0", "shortName": "LongCat 2.0", "slug": "longcat-2-0", "isReasoning": true, "deprecated": false, "modelCreatorName": "LongCat", "intelligenceIndex": 19.1123256610623, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.05898990881433253, "terminalBench40": 0, "terminalBench21": 0.50187265917603, "terminalbenchHard": null, "scicode": 0.363425925925926, "hle": 0.336886005560704, "critpt": 0.0257142857142857, "lcr": 0.65, "gpqa": 0.77979797979798, "omniscience": -23.416666666666668, "omniscienceAccuracy": 0.29633333333333334, "omniscienceNonHallucination": 0.24609189957366173, "gdpvalNormalized": 0.17809500000000003, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.006, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "Nova Lite", "shortName": "Nova Lite", "slug": "nova-lite", "isReasoning": false, "deprecated": true, "modelCreatorName": "Amazon", "intelligenceIndex": 6.66571545482029, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0429, "critpt": 0, "lcr": 0.19, "gpqa": 0.433333333333333, "omniscience": -42.166666666666664, "omniscienceAccuracy": 0.098, "omniscienceNonHallucination": 0.42387287509238725, "gdpvalNormalized": null, "price1mInputTokens": 0.06, "price1mOutputTokens": 0.24, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 300000, "medianOutputTokensPerSecond": 168.994288447145, "medianTimeToFirstTokenSeconds": 0.962761634499998, "isOpenWeights": false }, { "name": "Seed-OSS-36B-Instruct", "shortName": "Seed-OSS-36B-Instruct", "slug": "seed-oss-36b-instruct", "isReasoning": true, "deprecated": true, "modelCreatorName": "ByteDance Seed", "intelligenceIndex": 12.099257907924, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0987025023169602, "critpt": 0, "lcr": 0.613333333333333, "gpqa": 0.726262626262626, "omniscience": -52.46666666666667, "omniscienceAccuracy": 0.182, "omniscienceNonHallucination": 0.13610431947840262, "gdpvalNormalized": null, "price1mInputTokens": 0.21, "price1mOutputTokens": 0.57, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 512000, "medianOutputTokensPerSecond": 40.9379493487897, "medianTimeToFirstTokenSeconds": 2.94314167950003, "isOpenWeights": true }, { "name": "GPT-4.5 (Preview)", "shortName": "GPT-4.5 (Preview)", "slug": "gpt-4-5", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 9.57619358566713, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "GPT-5 Codex (High)", "shortName": "GPT-5 Codex (high)", "slug": "gpt-5-codex", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 24.8818938471151, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.378787878787879, "scicode": null, "hle": 0.278498609823911, "critpt": 0.0514285714285714, "lcr": 0.716666666666667, "gpqa": 0.837373737373737, "omniscience": -7.533333333333333, "omniscienceAccuracy": 0.387, "omniscienceNonHallucination": 0.24578575312669926, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "isOpenWeights": false }, { "name": "GLM-4.5-Air", "shortName": "GLM-4.5-Air", "slug": "glm-4-5-air", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 11.0889137659922, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.204545454545455, "scicode": null, "hle": 0.0704355885078777, "critpt": 0, "lcr": 0.466666666666667, "gpqa": 0.733333333333333, "omniscience": -61.53333333333333, "omniscienceAccuracy": 0.1625, "omniscienceNonHallucination": 0.07124378109452734, "gdpvalNormalized": null, "price1mInputTokens": 0.17, "price1mOutputTokens": 0.98, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 76.8989836532708, "medianTimeToFirstTokenSeconds": 2.44876498550013, "isOpenWeights": true }, { "name": "Qwen3.5 2B (Reasoning)", "shortName": "Qwen3.5 2B", "slug": "qwen3-5-2b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.94413664133259, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0299625468164794, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.025949953660797, "critpt": 0, "lcr": 0.21, "gpqa": 0.455555555555556, "omniscience": -60.18333333333333, "omniscienceAccuracy": 0.08066666666666666, "omniscienceNonHallucination": 0.2576142131979695, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "MiMo-V2-Pro", "shortName": "MiMo-V2-Pro", "slug": "mimo-v2-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 28.6442401425214, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.409090909090909, "scicode": null, "hle": 0.304448563484708, "critpt": 0.00285714285714286, "lcr": 0.683333333333333, "gpqa": 0.86969696969697, "omniscience": 4.616666666666666, "omniscienceAccuracy": 0.26616666666666666, "omniscienceNonHallucination": 0.700204406086759, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Tri-21B-Think", "shortName": "Tri-21B-Think", "slug": "tri-21b-think-v0-5", "isReasoning": true, "deprecated": false, "modelCreatorName": "Trillion Labs", "intelligenceIndex": 8.98704047108094, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0593141797961075, "critpt": 0.00285714285714286, "lcr": 0.156666666666667, "gpqa": 0.601010101010101, "omniscience": -62.55, "omniscienceAccuracy": 0.12333333333333334, "omniscienceNonHallucination": 0.1458174904942966, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "DeepSeek V4 Pro 0424 (High)", "shortName": "DeepSeek V4 Pro (high)", "slug": "deepseek-v4-pro-0424-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 30.1054806103209, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.647940074906367, "terminalbenchHard": 0.416666666666667, "scicode": null, "hle": 0.352177942539388, "critpt": 0.1, "lcr": 0.703333333333333, "gpqa": 0.905050505050505, "omniscience": -10.566666666666666, "omniscienceAccuracy": 0.414, "omniscienceNonHallucination": 0.11319681456200226, "gdpvalNormalized": 0.32041, "price1mInputTokens": 0.435, "price1mOutputTokens": 0.87, "cacheHitPrice": 0.0036, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 85.3264051065994, "medianTimeToFirstTokenSeconds": 1.69185090199994, "isOpenWeights": true }, { "name": "Granite 4.0 H Small", "shortName": "Granite 4.0 H Small", "slug": "granite-4-0-h-small", "isReasoning": false, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 6.04930220974964, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0379981464318814, "critpt": 0, "lcr": 0.113333333333333, "gpqa": 0.416161616161616, "omniscience": -61.016666666666666, "omniscienceAccuracy": 0.14283333333333334, "omniscienceNonHallucination": 0.12152440209994164, "gdpvalNormalized": null, "price1mInputTokens": 0.06, "price1mOutputTokens": 0.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 12.4202471784284, "medianTimeToFirstTokenSeconds": 39.135507701, "isOpenWeights": true }, { "name": "Qwen3 30B A3B 2507 (Reasoning)", "shortName": "Qwen3 30B A3B 2507", "slug": "qwen3-30b-a3b-2507-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.80537532870426, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.0765560533905436, "terminalBench40": 0, "terminalBench21": 0.0149812734082397, "terminalbenchHard": 0.053030303030303, "scicode": 0.329861111111111, "hle": 0.102873030583874, "critpt": 0.00285714285714286, "lcr": 0.613333333333333, "gpqa": 0.707070707070707, "omniscience": -56.13333333333333, "omniscienceAccuracy": 0.163, "omniscienceNonHallucination": 0.13460772600557547, "gdpvalNormalized": 0, "price1mInputTokens": 0.2, "price1mOutputTokens": 2.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 132.015253484745, "medianTimeToFirstTokenSeconds": 2.40189343399987, "isOpenWeights": true }, { "name": "Mistral Large (Feb '24)", "shortName": "Mistral Large (Feb)", "slug": "mistral-large", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.76443460315821, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.035, "critpt": null, "lcr": null, "gpqa": 0.350505050505051, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 4, "price1mOutputTokens": 12, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": false }, { "name": "Qwen3.8 Max", "shortName": "Qwen3.8 Max", "slug": "qwen3-8-max-0803", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 40.1528623585941, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.6699848505343478, "terminalBench40": 0.186868686868687, "terminalBench21": 0.812734082397004, "terminalbenchHard": null, "scicode": 0.532407407407407, "hle": 0.430491195551437, "critpt": 0.2, "lcr": 0.783333333333333, "gpqa": 0.927272727272727, "omniscience": 3.4, "omniscienceAccuracy": 0.3185, "omniscienceNonHallucination": 0.5825385179750551, "gdpvalNormalized": 0.547765, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.25, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 38.630492111592, "medianTimeToFirstTokenSeconds": 2.73924980399988, "isOpenWeights": false }, { "name": "Grok 4.3 (Medium)", "shortName": "Grok 4.3 (medium)", "slug": "grok-4-3-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 24.7816938151637, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.303030303030303, "scicode": null, "hle": 0.299814643188137, "critpt": 0.0485714285714286, "lcr": 0.75, "gpqa": 0.88989898989899, "omniscience": 16.7, "omniscienceAccuracy": 0.2876666666666667, "omniscienceNonHallucination": 0.8306036499766027, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.2, "cacheWritePrice": 1.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 109.096983037189, "medianTimeToFirstTokenSeconds": 14.923576059, "isOpenWeights": false }, { "name": "GPT-5 (Medium)", "shortName": "GPT-5 (medium)", "slug": "gpt-5-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 22.869335846124, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.378787878787879, "scicode": null, "hle": 0.253938832252085, "critpt": 0, "lcr": 0.76, "gpqa": 0.841750841750842, "omniscience": -10.866666666666667, "omniscienceAccuracy": 0.3948333333333333, "omniscienceNonHallucination": 0.16799779675020654, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 75.6163254747146, "medianTimeToFirstTokenSeconds": 39.604762958, "isOpenWeights": false }, { "name": "Grok 4", "shortName": "Grok 4", "slug": "grok-4", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 22.4866809354881, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.378787878787879, "scicode": null, "hle": 0.266913809082484, "critpt": 0.02, "lcr": 0.68, "gpqa": 0.876767676767677, "omniscience": 2.1, "omniscienceAccuracy": 0.4048333333333333, "omniscienceNonHallucination": 0.35508260991318963, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Sonar", "shortName": "Sonar", "slug": "sonar", "isReasoning": false, "deprecated": true, "modelCreatorName": "Perplexity", "intelligenceIndex": 7.70971507200974, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0488, "critpt": null, "lcr": null, "gpqa": 0.470707070707071, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 127000, "isOpenWeights": false }, { "name": "Llama 3.2 Instruct 11B (Vision)", "shortName": "Llama 3.2 11B (Vision)", "slug": "llama-3-2-instruct-11b-vision", "isReasoning": false, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 5.40802026647316, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0553, "critpt": 0, "lcr": 0.126666666666667, "gpqa": 0.221212121212121, "omniscience": -62.6, "omniscienceAccuracy": 0.106, "omniscienceNonHallucination": 0.18120805369127513, "gdpvalNormalized": null, "price1mInputTokens": 0.345, "price1mOutputTokens": 0.345, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 15.2680961057345, "medianTimeToFirstTokenSeconds": 3.03310276450001, "isOpenWeights": true }, { "name": "DeepSeek R1 0528 Qwen3 8B", "shortName": "DeepSeek R1 0528 Qwen3 8B", "slug": "deepseek-r1-qwen3-8b", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 8.07891570839329, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0593141797961075, "critpt": 0, "lcr": 0.15, "gpqa": 0.612121212121212, "omniscience": -63.61666666666667, "omniscienceAccuracy": 0.122, "omniscienceNonHallucination": 0.13648443432042523, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Qwen3.6 Max Preview", "shortName": "Qwen3.6 Max Preview", "slug": "qwen3-6-max", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 28.3748690464685, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.439393939393939, "scicode": null, "hle": 0.308155699721965, "critpt": 0.0370612244897959, "lcr": 0.806666666666667, "gpqa": 0.887878787878788, "omniscience": 9.166666666666666, "omniscienceAccuracy": 0.37866666666666665, "omniscienceNonHallucination": 0.5380901287553648, "gdpvalNormalized": null, "price1mInputTokens": 1.3, "price1mOutputTokens": 7.8, "cacheHitPrice": 0.13, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 69.1232438594986, "medianTimeToFirstTokenSeconds": 2.97260116949999, "isOpenWeights": false }, { "name": "LongCat Flash Lite", "shortName": "LongCat Flash Lite", "slug": "longcat-flash-lite", "isReasoning": false, "deprecated": false, "modelCreatorName": "LongCat", "intelligenceIndex": 11.4709079965186, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.106060606060606, "scicode": null, "hle": 0.0583873957367933, "critpt": 0, "lcr": 0.283333333333333, "gpqa": 0.636363636363636, "omniscience": -70.06666666666666, "omniscienceAccuracy": 0.133, "omniscienceNonHallucination": 0.03844675124951946, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Tri-21B-think Preview", "shortName": "Tri-21B-think Preview", "slug": "tri-21b-think-preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "Trillion Labs", "intelligenceIndex": 9.59191548491904, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0579240037071362, "critpt": 0, "lcr": 0.216666666666667, "gpqa": 0.538383838383838, "omniscience": -62.3, "omniscienceAccuracy": 0.101, "omniscienceNonHallucination": 0.1946607341490545, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Nova 2.0 Lite (High)", "shortName": "Nova 2.0 Lite (high)", "slug": "nova-2-0-lite-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 13.3740793005281, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.161048689138577, "terminalbenchHard": 0.166666666666667, "scicode": null, "hle": 0.11631139944393, "critpt": 0.00285714285714286, "lcr": 0.603333333333333, "gpqa": 0.811111111111111, "omniscience": -54.983333333333334, "omniscienceAccuracy": 0.19033333333333333, "omniscienceNonHallucination": 0.08583779333058872, "gdpvalNormalized": 0, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 182.514229291011, "medianTimeToFirstTokenSeconds": 12.6329232835, "isOpenWeights": false }, { "name": "Gemini 2.5 Pro Preview (Mar' 25)", "shortName": "Gemini 2.5 Pro (Mar)", "slug": "gemini-2-5-pro-03-25", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 14.9609881454387, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.180259499536608, "critpt": null, "lcr": null, "gpqa": 0.836363636363636, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Solar Mini 4", "shortName": "Solar Mini 4", "slug": "solar-mini4", "isReasoning": true, "deprecated": false, "modelCreatorName": "Upstage", "intelligenceIndex": 24.0522018844171, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.36336112521859043, "terminalBench40": 0.0101010101010101, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.475694444444444, "hle": 0.257645968489342, "critpt": 0.0142857142857143, "lcr": 0.833333333333333, "gpqa": null, "omniscience": -10.816666666666666, "omniscienceAccuracy": 0.18416666666666667, "omniscienceNonHallucination": 0.6416751787538304, "gdpvalNormalized": 0.285915, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 1048576, "medianOutputTokensPerSecond": 204.068813947352, "medianTimeToFirstTokenSeconds": 1.48549577899993, "isOpenWeights": false }, { "name": "Nemotron 3 Super 120B A12B (Reasoning)", "shortName": "Nemotron 3 Super", "slug": "nvidia-nemotron-3-super-120b-a12b", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 12.8287575290979, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.6390663672616101, "terminalBench40": 0, "terminalBench21": 0.385767790262172, "terminalbenchHard": 0.287878787878788, "scicode": 0.362268518518519, "hle": 0.207599629286376, "critpt": 0.0314285714285714, "lcr": 0.656666666666667, "gpqa": 0.8, "omniscience": -41.5, "omniscienceAccuracy": 0.24316666666666667, "omniscienceNonHallucination": 0.13036776040519704, "gdpvalNormalized": 0, "price1mInputTokens": 0.3, "price1mOutputTokens": 0.9, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 164.518903546696, "medianTimeToFirstTokenSeconds": 1.31421257900001, "isOpenWeights": true }, { "name": "GPT-6 Luna (Non-reasoning)", "shortName": "GPT-6 Luna (non-reasoning)", "slug": "gpt-6-luna-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 18.2617465935701, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.01146983124544933, "terminalBench40": 0.0151515151515152, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.430555555555556, "hle": 0.0857275254865616, "critpt": 0.0114285714285714, "lcr": 0.396666666666667, "gpqa": null, "omniscience": -21.9, "omniscienceAccuracy": 0.3185, "omniscienceNonHallucination": 0.2112986060161408, "gdpvalNormalized": 0.267015, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.01, "cacheWritePrice": 0.125, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 132.256769665819, "medianTimeToFirstTokenSeconds": 0.806585144999971, "isOpenWeights": false }, { "name": "MiniCPM5-2B", "shortName": "MiniCPM5-2B", "slug": "minicpm5-2b", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenBMB", "intelligenceIndex": 12.463354831996, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.0861423220973783, "terminalbenchHard": null, "scicode": 0.262731481481481, "hle": 0.0894346617238184, "critpt": 0.00285714285714286, "lcr": 0.593333333333333, "gpqa": 0.702020202020202, "omniscience": -11.65, "omniscienceAccuracy": 0.08416666666666667, "omniscienceNonHallucination": 0.780891719745223, "gdpvalNormalized": 0.09713499999999999, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Grok 4.20 0309 v2 (Non-reasoning)", "shortName": "Grok 4.20 0309 v2 (non-reasoning)", "slug": "grok-4-20-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 14.2010039023924, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.166666666666667, "scicode": null, "hle": 0.279425393883225, "critpt": 0.00285714285714286, "lcr": 0.233333333333333, "gpqa": 0.775757575757576, "omniscience": -44.63333333333333, "omniscienceAccuracy": 0.2668333333333333, "omniscienceNonHallucination": 0.027278927028870248, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 2000000, "medianOutputTokensPerSecond": 88.7245790042118, "medianTimeToFirstTokenSeconds": 0.634296690499923, "isOpenWeights": false }, { "name": "Qwen3.5 4B (Reasoning)", "shortName": "Qwen3.5 4B", "slug": "qwen3-5-4b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 13.1248174602542, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.258426966292135, "terminalbenchHard": 0.181818181818182, "scicode": null, "hle": 0.0991658943466172, "critpt": 0, "lcr": 0.63, "gpqa": 0.770707070707071, "omniscience": -58.35, "omniscienceAccuracy": 0.15116666666666667, "omniscienceNonHallucination": 0.13449833104260756, "gdpvalNormalized": null, "price1mInputTokens": 0.03, "price1mOutputTokens": 0.15, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 30.5185813374024, "medianTimeToFirstTokenSeconds": 0.870860545000028, "isOpenWeights": true }, { "name": "Magistral Medium 1", "shortName": "Magistral Medium 1", "slug": "magistral-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 9.05394062240647, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0909090909090909, "scicode": null, "hle": 0.0982391102873031, "critpt": 0.00285714285714286, "lcr": 0, "gpqa": 0.678787878787879, "omniscience": -26.933333333333334, "omniscienceAccuracy": 0.20333333333333334, "omniscienceNonHallucination": 0.4066945606694561, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 40000, "isOpenWeights": false }, { "name": "Kimi K2 0905", "shortName": "Kimi K2 0905", "slug": "kimi-k2-0905", "isReasoning": false, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 15.2941620102232, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.234848484848485, "scicode": null, "hle": 0.0639481000926784, "critpt": 0, "lcr": 0.53, "gpqa": 0.766666666666667, "omniscience": -26.583333333333332, "omniscienceAccuracy": 0.25416666666666665, "omniscienceNonHallucination": 0.30279329608938543, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 52.6012489763269, "medianTimeToFirstTokenSeconds": 1.37383396699998, "isOpenWeights": true }, { "name": "Qwen3.8 27B (Medium)", "shortName": "Qwen3.8 27B (medium)", "slug": "qwen3-8-27b-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 27.5508228231586, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.133063774030521, "terminalBench40": 0.0505050505050505, "terminalBench21": 0.651685393258427, "terminalbenchHard": null, "scicode": 0.390046296296296, "hle": 0.140871177015755, "critpt": 0, "lcr": 0.796666666666667, "gpqa": 0.845454545454545, "omniscience": -36.15, "omniscienceAccuracy": 0.1835, "omniscienceNonHallucination": 0.33251684017146355, "gdpvalNormalized": 0.434245, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.1, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 48.8808040268763, "medianTimeToFirstTokenSeconds": 3.88132494000001, "isOpenWeights": true }, { "name": "Motif-2-12.7B-Reasoning", "shortName": "Motif-2-12.7B", "slug": "motif-2-12-7b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Motif Technologies", "intelligenceIndex": 9.18748556817538, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.08758109360519, "critpt": 0, "lcr": 0.146666666666667, "gpqa": 0.694949494949495, "omniscience": -61.68333333333333, "omniscienceAccuracy": 0.15166666666666667, "omniscienceNonHallucination": 0.09410609037328099, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Qwen3.6 35B A3B (Non-reasoning)", "shortName": "Qwen3.6 35B A3B (non-reasoning)", "slug": "qwen3-6-35b-a3b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 15.2320772647744, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.415730337078652, "terminalbenchHard": 0.257575757575758, "scicode": null, "hle": 0.139017608897127, "critpt": 0, "lcr": 0.643333333333333, "gpqa": 0.817171717171717, "omniscience": -60.016666666666666, "omniscienceAccuracy": 0.16716666666666666, "omniscienceNonHallucination": 0.07864718831298778, "gdpvalNormalized": 0.16867500000000002, "price1mInputTokens": 0.375, "price1mOutputTokens": 2.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 139.309552638662, "medianTimeToFirstTokenSeconds": 2.1197436435, "isOpenWeights": true }, { "name": "Ministral 3 3B", "shortName": "Ministral 3 3B", "slug": "ministral-3-3b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 4.83760020546443, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.007846860809830004, "terminalBench40": 0, "terminalBench21": 0, "terminalbenchHard": 0, "scicode": 0.152777777777778, "hle": 0.0537534754402224, "critpt": 0, "lcr": 0.17, "gpqa": 0.357575757575758, "omniscience": -64, "omniscienceAccuracy": 0.08983333333333333, "omniscienceNonHallucination": 0.19813221021790883, "gdpvalNormalized": 0, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.1, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 198.309229836907, "medianTimeToFirstTokenSeconds": 0.681828317499964, "isOpenWeights": true }, { "name": "LFM2 8B A1B", "shortName": "LFM2 8B A1B", "slug": "lfm2-8b-a1b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Liquid AI", "intelligenceIndex": 4.92956933760184, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0491195551436515, "critpt": 0, "lcr": null, "gpqa": 0.344444444444444, "omniscience": -76.5, "omniscienceAccuracy": 0.08216666666666667, "omniscienceNonHallucination": 0.07699291810423103, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Gemini 2.5 Flash Preview (Reasoning)", "shortName": "Gemini 2.5 Flash (Apr)", "slug": "gemini-2-5-flash-reasoning-04-2025", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 11.6539041327006, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.121, "critpt": null, "lcr": null, "gpqa": 0.697979797979798, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Qwen3.7 Max", "shortName": "Qwen3.7 Max", "slug": "qwen3-7-max", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 29.4571634255623, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.1472961597972207, "terminalBench40": 0.0151515151515152, "terminalBench21": 0.745318352059925, "terminalbenchHard": 0.507575757575758, "scicode": 0.49537037037037, "hle": 0.405004633920297, "critpt": 0.134285714285714, "lcr": 0.79, "gpqa": 0.923232323232323, "omniscience": 13.483333333333333, "omniscienceAccuracy": 0.31133333333333335, "omniscienceNonHallucination": 0.7437076476282671, "gdpvalNormalized": 0.30713999999999997, "price1mInputTokens": 2.5, "price1mOutputTokens": 7.5, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 202.16069583885, "medianTimeToFirstTokenSeconds": 2.23291775900003, "isOpenWeights": false }, { "name": "GLM-4.6 (Reasoning)", "shortName": "GLM-4.6", "slug": "glm-4-6-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 18.5297479165936, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.49438202247191, "terminalbenchHard": 0.25, "scicode": null, "hle": 0.144578313253012, "critpt": 0.0114285714285714, "lcr": 0.54, "gpqa": 0.77979797979798, "omniscience": -41.88333333333333, "omniscienceAccuracy": 0.2688333333333333, "omniscienceNonHallucination": 0.05949395942557556, "gdpvalNormalized": 0.12297500000000003, "price1mInputTokens": 0.55, "price1mOutputTokens": 2.2, "cacheHitPrice": 0.11, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 30.3116696396012, "medianTimeToFirstTokenSeconds": 4.7379809869999, "isOpenWeights": true }, { "name": "GPT-4", "shortName": "GPT-4", "slug": "gpt-4", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 6.70159308543366, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.349494949494949, "omniscience": -33.833333333333336, "omniscienceAccuracy": 0.21, "omniscienceNonHallucination": 0.30590717299578063, "gdpvalNormalized": 0, "price1mInputTokens": 30, "price1mOutputTokens": 60, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": false }, { "name": "Gemini 2.5 Flash (Non-reasoning)", "shortName": "Gemini 2.5 Flash (non-reasoning)", "slug": "gemini-2-5-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 9.85128154382851, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.121212121212121, "scicode": null, "hle": 0.0472659870250232, "critpt": 0.0142857142857143, "lcr": 0.499, "gpqa": 0.682828282828283, "omniscience": -42.56666666666667, "omniscienceAccuracy": 0.2613333333333333, "omniscienceNonHallucination": 0.06994584837545126, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.03, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 188.534541543001, "medianTimeToFirstTokenSeconds": 0.450865774000021, "isOpenWeights": false }, { "name": "Llama 2 Chat 13B", "shortName": "Llama 2 Chat 13B", "slug": "llama-2-chat-13b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 5.30069788678423, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0477, "critpt": null, "lcr": null, "gpqa": 0.321212121212121, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "GPT-5.5 (Medium)", "shortName": "GPT-5.5 (medium)", "slug": "gpt-5-5-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 33.8043604432432, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.9016588021523384, "terminalBench40": 0.0505050505050505, "terminalBench21": 0.805243445692884, "terminalbenchHard": 0.575757575757576, "scicode": 0.545138888888889, "hle": 0.42354031510658, "critpt": 0.185714285714286, "lcr": 0.83, "gpqa": 0.926262626262626, "omniscience": 18.1, "omniscienceAccuracy": 0.568, "omniscienceNonHallucination": 0.10416666666666663, "gdpvalNormalized": 0.359215, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 922000, "medianOutputTokensPerSecond": 83.5125070494008, "medianTimeToFirstTokenSeconds": 6.80746101850021, "isOpenWeights": false }, { "name": "Hermes 4 - Llama-3.1 70B (Reasoning)", "shortName": "Hermes 4 70B", "slug": "hermes-4-llama-3-1-70b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Nous Research", "intelligenceIndex": 7.90579275030455, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.08758109360519, "critpt": 0, "lcr": 0.0966666666666667, "gpqa": 0.698989898989899, "omniscience": -49.36666666666667, "omniscienceAccuracy": 0.2355, "omniscienceNonHallucination": 0.04621757139742755, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Step 3.7 Flash", "shortName": "Step 3.7 Flash", "slug": "step-3-7-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "StepFun", "intelligenceIndex": 19.4812572685302, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.393258426966292, "terminalbenchHard": 0.356060606060606, "scicode": 0.438657407407407, "hle": 0.214087117701576, "critpt": 0.0228571428571429, "lcr": 0.736666666666667, "gpqa": 0.809090909090909, "omniscience": -37.28333333333333, "omniscienceAccuracy": 0.25783333333333336, "omniscienceNonHallucination": 0.1502357960925219, "gdpvalNormalized": 0.172415, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.15, "cacheHitPrice": 0.04, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 182.92548403666, "medianTimeToFirstTokenSeconds": 2.66042430549995, "isOpenWeights": true }, { "name": "Ling-3.0-flash-Fin", "shortName": "Ling-3.0-flash-Fin", "slug": "ling-3-0-flash-fin", "isReasoning": true, "deprecated": false, "modelCreatorName": "InclusionAI", "intelligenceIndex": 22.5965828940594, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.625468164794007, "terminalbenchHard": null, "scicode": 0.423611111111111, "hle": 0.22613531047266, "critpt": 0.0257142857142857, "lcr": 0.736666666666667, "gpqa": null, "omniscience": -14.583333333333334, "omniscienceAccuracy": 0.17866666666666667, "omniscienceNonHallucination": 0.6049107142857143, "gdpvalNormalized": 0.29647, "price1mInputTokens": 0.075, "price1mOutputTokens": 0.22, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 302.989013689122, "medianTimeToFirstTokenSeconds": 1.74881907600002, "isOpenWeights": true }, { "name": "DeepSeek V3.2 (Non-reasoning)", "shortName": "DeepSeek V3.2 (non-reasoning)", "slug": "deepseek-v3-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 16.043537719683, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.325757575757576, "scicode": null, "hle": 0.112140871177016, "critpt": 0.00857142857142857, "lcr": 0.456666666666667, "gpqa": 0.750505050505051, "omniscience": -46.88333333333333, "omniscienceAccuracy": 0.24, "omniscienceNonHallucination": 0.06732456140350873, "gdpvalNormalized": null, "price1mInputTokens": 0.28, "price1mOutputTokens": 0.42, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3 VL 32B Instruct", "shortName": "Qwen3 VL 32B", "slug": "qwen3-vl-32b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.38699329476692, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0681186283595922, "critpt": 0, "lcr": null, "gpqa": 0.670707070707071, "omniscience": -63.36666666666667, "omniscienceAccuracy": 0.14833333333333334, "omniscienceNonHallucination": 0.08180039138943251, "gdpvalNormalized": null, "price1mInputTokens": 0.16, "price1mOutputTokens": 0.64, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 67.2249733167685, "medianTimeToFirstTokenSeconds": 2.65428834650004, "isOpenWeights": true }, { "name": "GPT-5.2 (Non-reasoning)", "shortName": "GPT-5.2 (non-reasoning)", "slug": "gpt-5-2-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 16.97598252351, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.318181818181818, "scicode": null, "hle": 0.0801668211306766, "critpt": 0.00571428571428571, "lcr": 0.456666666666667, "gpqa": 0.712121212121212, "omniscience": -12.25, "omniscienceAccuracy": 0.3085, "omniscienceNonHallucination": 0.37671728127259585, "gdpvalNormalized": null, "price1mInputTokens": 1.75, "price1mOutputTokens": 14, "cacheHitPrice": 0.175, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 67.2319844904394, "medianTimeToFirstTokenSeconds": 0.856808193000006, "isOpenWeights": false }, { "name": "Gemma 4 26B A4B (Reasoning)", "shortName": "Gemma 4 26B A4B", "slug": "gemma-4-26b-a4b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 16.671930040086, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.389513108614232, "terminalbenchHard": 0.1364, "scicode": 0.400462962962963, "hle": 0.193234476367006, "critpt": 0, "lcr": 0.656666666666667, "gpqa": 0.791919191919192, "omniscience": -50.833333333333336, "omniscienceAccuracy": 0.191, "omniscienceNonHallucination": 0.13555830243098477, "gdpvalNormalized": 0.026464999999999975, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.37, "cacheHitPrice": 0.060000000000000005, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "DeepSeek V4 Pro 0424 (Non-reasoning)", "shortName": "DeepSeek V4 Pro (non-reasoning)", "slug": "deepseek-v4-pro-0424-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 20.8344651194768, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.363636363636364, "scicode": null, "hle": 0.082483781278962, "critpt": 0.00857142857142857, "lcr": 0.53, "gpqa": 0.717171717171717, "omniscience": -29.866666666666667, "omniscienceAccuracy": 0.30866666666666664, "omniscienceNonHallucination": 0.12150433944069428, "gdpvalNormalized": null, "price1mInputTokens": 0.435, "price1mOutputTokens": 0.87, "cacheHitPrice": 0.0036, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 78.497756769288, "medianTimeToFirstTokenSeconds": 1.67119331000004, "isOpenWeights": true }, { "name": "NVIDIA Nemotron Nano 12B v2 VL (Reasoning)", "shortName": "NVIDIA Nemotron Nano 12B v2 VL", "slug": "nvidia-nemotron-nano-12b-v2-vl-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.47796854824097, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0551436515291937, "critpt": 0, "lcr": 0.41, "gpqa": 0.571717171717172, "omniscience": -64.23333333333333, "omniscienceAccuracy": 0.14616666666666667, "omniscienceNonHallucination": 0.07651766543041183, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-5.6 Sol (Medium)", "shortName": "GPT-5.6 Sol (medium)", "slug": "gpt-5-6-sol-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 39.2363585889147, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5049926304382959, "terminalBench40": 0.146464646464646, "terminalBench21": 0.861423220973783, "terminalbenchHard": 0.628787878787879, "scicode": 0.574074074074074, "hle": 0.422150139017609, "critpt": 0.228571428571429, "lcr": 0.803333333333333, "gpqa": 0.926262626262626, "omniscience": 19.4, "omniscienceAccuracy": 0.5775, "omniscienceNonHallucination": 0.09230769230769231, "gdpvalNormalized": 0.451475, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.4, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 70.3430535075343, "medianTimeToFirstTokenSeconds": 6.59239555999988, "isOpenWeights": false }, { "name": "GPT-5.5 (Non-reasoning)", "shortName": "GPT-5.5 (non-reasoning)", "slug": "gpt-5-5-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 23.1674381348311, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.610486891385768, "terminalbenchHard": 0.492424242424242, "scicode": null, "hle": 0.137164040778499, "critpt": 0.0142857142857143, "lcr": 0.64, "gpqa": 0.767676767676768, "omniscience": -4.75, "omniscienceAccuracy": 0.4563333333333333, "omniscienceNonHallucination": 0.07326793378295526, "gdpvalNormalized": 0.22852999999999998, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 922000, "medianOutputTokensPerSecond": 77.2157645658437, "medianTimeToFirstTokenSeconds": 0.877006933500041, "isOpenWeights": false }, { "name": "Qwen3.5 Omni Flash", "shortName": "Qwen3.5 Omni Flash", "slug": "qwen3-5-omni-flash", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 12.4740069826994, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0759962928637627, "critpt": 0, "lcr": 0.52, "gpqa": 0.742424242424242, "omniscience": -65.06666666666666, "omniscienceAccuracy": 0.1465, "omniscienceNonHallucination": 0.06600273384104671, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 219.137254451062, "medianTimeToFirstTokenSeconds": 1.83819265699992, "isOpenWeights": false }, { "name": "GPT-6 Luna (Max)", "shortName": "GPT-6 Luna (max)", "slug": "gpt-6-luna", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 37.2559686869738, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.06792906856485216, "terminalBench40": 0.126262626262626, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.546296296296296, "hle": 0.385078776645042, "critpt": 0.194285714285714, "lcr": 0.833333333333333, "gpqa": null, "omniscience": 0.65, "omniscienceAccuracy": 0.43783333333333335, "omniscienceNonHallucination": 0.23273050696709163, "gdpvalNormalized": 0.43354499999999996, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.01, "cacheWritePrice": 0.125, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 127.032013454234, "medianTimeToFirstTokenSeconds": 123.885851666, "isOpenWeights": false }, { "name": "GLM-4.7 (Reasoning)", "shortName": "GLM-4.7", "slug": "glm-4-7", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 22.2402659332875, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.453183520599251, "terminalbenchHard": 0.318181818181818, "scicode": null, "hle": 0.27386468952734, "critpt": 0.0171020408163265, "lcr": 0.71, "gpqa": 0.858585858585858, "omniscience": -36.43333333333333, "omniscienceAccuracy": 0.2931666666666667, "omniscienceNonHallucination": 0.06979485970290022, "gdpvalNormalized": 0.24989499999999998, "price1mInputTokens": 0.6, "price1mOutputTokens": 2.2, "cacheHitPrice": 0.45, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 67.0082777268343, "medianTimeToFirstTokenSeconds": 1.29518703450002, "isOpenWeights": true }, { "name": "Nova 2.0 Lite (Low)", "shortName": "Nova 2.0 Lite (low)", "slug": "nova-2-0-lite-reasoning-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 11.8041280807831, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0403151065801668, "critpt": 0, "lcr": 0.543333333333333, "gpqa": 0.697979797979798, "omniscience": -54.1, "omniscienceAccuracy": 0.1765, "omniscienceNonHallucination": 0.12871888281724342, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 191.653666030509, "medianTimeToFirstTokenSeconds": 9.3280531025, "isOpenWeights": false }, { "name": "Magistral Small 1.2", "shortName": "Magistral Small 1.2", "slug": "magistral-small-2509", "isReasoning": true, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 8.59366235978288, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0449438202247191, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0644114921223355, "critpt": 0.00285714285714286, "lcr": 0.193333333333333, "gpqa": 0.662626262626263, "omniscience": -65.1, "omniscienceAccuracy": 0.1345, "omniscienceNonHallucination": 0.09243212016175617, "gdpvalNormalized": 0, "price1mInputTokens": 0.5, "price1mOutputTokens": 1.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemma 4 26B A4B (Non-reasoning)", "shortName": "Gemma 4 26B A4B (non-reasoning)", "slug": "gemma-4-26b-a4b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 13.1303772801787, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.25, "scicode": null, "hle": 0.114921223354958, "critpt": 0, "lcr": 0.423333333333333, "gpqa": 0.714141414141414, "omniscience": -62.31666666666667, "omniscienceAccuracy": 0.15483333333333332, "omniscienceNonHallucination": 0.07947150463419439, "gdpvalNormalized": null, "price1mInputTokens": 0.13, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.1, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 112.305157188542, "medianTimeToFirstTokenSeconds": 1.0402467190001, "isOpenWeights": true }, { "name": "Claude Opus 5 (Adaptive Reasoning, High Effort)", "shortName": "Claude Opus 5 (high)", "slug": "claude-opus-5-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 48.1219182595908, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 3.6132670078004714, "terminalBench40": 0.45959595959596, "terminalBench21": 0.876404494382023, "terminalbenchHard": null, "scicode": 0.554398148148148, "hle": 0.528266913809083, "critpt": 0.282857142857143, "lcr": 0.79, "gpqa": 0.937373737373737, "omniscience": 33.71666666666667, "omniscienceAccuracy": 0.5888333333333333, "omniscienceNonHallucination": 0.38792055127685443, "gdpvalNormalized": 0.5405, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 53.3736410718684, "medianTimeToFirstTokenSeconds": 19.959415992, "isOpenWeights": false }, { "name": "Ministral 3 14B", "shortName": "Ministral 3 14B", "slug": "ministral-3-14b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 6.04436817215002, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.01943640228162961, "terminalBench40": 0, "terminalBench21": 0.0973782771535581, "terminalbenchHard": 0.0454545454545455, "scicode": 0.238425925925926, "hle": 0.046339202965709, "critpt": 0, "lcr": 0.263333333333333, "gpqa": 0.571717171717172, "omniscience": -66.36666666666666, "omniscienceAccuracy": 0.13583333333333333, "omniscienceNonHallucination": 0.074831243972999, "gdpvalNormalized": 0, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.2, "cacheHitPrice": 0.02, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 74.2449015765375, "medianTimeToFirstTokenSeconds": 1.01887410300003, "isOpenWeights": true }, { "name": "K-EXAONE (Reasoning)", "shortName": "K-EXAONE", "slug": "k-exaone", "isReasoning": true, "deprecated": true, "modelCreatorName": "LG AI Research", "intelligenceIndex": 14.3683727563074, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.303370786516854, "terminalbenchHard": 0.227272727272727, "scicode": null, "hle": 0.139481000926784, "critpt": 0.0111836734693878, "lcr": 0.613333333333333, "gpqa": 0.782828282828283, "omniscience": -57.96666666666667, "omniscienceAccuracy": 0.1635, "omniscienceNonHallucination": 0.11157601115760107, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Llama 3.2 Instruct 3B", "shortName": "Llama 3.2 3B", "slug": "llama-3-2-instruct-3b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 5.70063694214945, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0535, "critpt": null, "lcr": 0.0433333333333333, "gpqa": 0.254545454545455, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 2.5 Flash-Lite Preview (Sep '25) (Non-reasoning)", "shortName": "Gemini 2.5 Flash-Lite (Sep) (non-reasoning)", "slug": "gemini-2-5-flash-lite-preview-09-2025", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 9.33681660061059, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0757575757575758, "scicode": null, "hle": 0.0509731232622799, "critpt": 0, "lcr": 0.496666666666667, "gpqa": 0.65050505050505, "omniscience": -42.56666666666667, "omniscienceAccuracy": 0.1415, "omniscienceNonHallucination": 0.3393515822170452, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Inkling Small", "shortName": "Inkling Small", "slug": "inkling-small", "isReasoning": true, "deprecated": false, "modelCreatorName": "Thinking Machines", "intelligenceIndex": 25.6582584411942, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.0894649973284348, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.550561797752809, "terminalbenchHard": null, "scicode": 0.496527777777778, "hle": 0.333178869323448, "critpt": 0.0828571428571429, "lcr": 0.756666666666667, "gpqa": 0.894949494949495, "omniscience": -8.933333333333334, "omniscienceAccuracy": 0.33166666666666667, "omniscienceNonHallucination": 0.37007481296758105, "gdpvalNormalized": 0.30428999999999995, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.06, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 187.800772189829, "medianTimeToFirstTokenSeconds": 2.0918537890002, "isOpenWeights": true }, { "name": "GPT-4.1 nano", "shortName": "GPT-4.1 nano", "slug": "gpt-4-1-nano", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 7.81796186868883, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0374531835205993, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0375347544022243, "critpt": 0, "lcr": 0.203333333333333, "gpqa": 0.512121212121212, "omniscience": -57.583333333333336, "omniscienceAccuracy": 0.13683333333333333, "omniscienceNonHallucination": 0.17435798416682757, "gdpvalNormalized": 0, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 133.067571550785, "medianTimeToFirstTokenSeconds": 0.709754856999979, "isOpenWeights": false }, { "name": "Llama 3.1 Nemotron Instruct 70B", "shortName": "Llama 3.1 Nemotron 70B", "slug": "llama-3-1-nemotron-instruct-70b", "isReasoning": false, "deprecated": true, "modelCreatorName": "NVIDIA", "intelligenceIndex": 6.9438219838506, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0419, "critpt": 0, "lcr": 0.0833333333333333, "gpqa": 0.464646464646465, "omniscience": -40.833333333333336, "omniscienceAccuracy": 0.17766666666666667, "omniscienceNonHallucination": 0.2873935954600729, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Llama 3.1 Instruct 8B", "shortName": "Llama 3.1 8B", "slug": "llama-3-1-instruct-8b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 6.93350756346201, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0149812734082397, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0529, "critpt": 0, "lcr": 0.18, "gpqa": 0.258585858585859, "omniscience": -30.883333333333333, "omniscienceAccuracy": 0.085, "omniscienceNonHallucination": 0.5695810564663024, "gdpvalNormalized": 0, "price1mInputTokens": 0.02, "price1mOutputTokens": 0.05, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 139.194870313857, "medianTimeToFirstTokenSeconds": 0.903213946000051, "isOpenWeights": true }, { "name": "LFM2 2.6B", "shortName": "LFM2 2.6B", "slug": "lfm2-2-6b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Liquid AI", "intelligenceIndex": 5.21069904506505, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0546802594995366, "critpt": 0, "lcr": 0, "gpqa": 0.306060606060606, "omniscience": -54.06666666666667, "omniscienceAccuracy": 0.05416666666666667, "omniscienceNonHallucination": 0.37110132158590303, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "DeepSeek V4.1 Flash (Non-reasoning)", "shortName": "DeepSeek V4.1 Flash (non-reasoning)", "slug": "deepseek-v4-1-flash-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "DeepSeek", "intelligenceIndex": 24.6733593924318, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.14670039195227308, "terminalBench40": 0.0555555555555556, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.355324074074074, "hle": 0.107970342910102, "critpt": 0.00285714285714286, "lcr": 0.553333333333333, "gpqa": null, "omniscience": -10.183333333333334, "omniscienceAccuracy": 0.2845, "omniscienceNonHallucination": 0.46005124621476823, "gdpvalNormalized": 0.413845, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.006, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 202.203043683669, "medianTimeToFirstTokenSeconds": 1.011459247, "isOpenWeights": true }, { "name": "Gemini 3.8 Flash (Medium)", "shortName": "Gemini 3.8 Flash (medium)", "slug": "gemini-3-8-flash-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 39.7740187061926, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.9310379374376306, "terminalBench40": 0.196969696969697, "terminalBench21": 0.838951310861423, "terminalbenchHard": null, "scicode": 0.550925925925926, "hle": 0.421223354958295, "critpt": 0.122857142857143, "lcr": 0.84, "gpqa": 0.935353535353535, "omniscience": 28.583333333333332, "omniscienceAccuracy": 0.5298333333333334, "omniscienceNonHallucination": 0.4810350939383198, "gdpvalNormalized": 0.45428999999999997, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.075, "cacheWritePrice": 0.75, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Gemma 4 E2B (Non-reasoning)", "shortName": "Gemma 4 E2B (non-reasoning)", "slug": "gemma-4-e2b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 6.47543520107148, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0472659870250232, "critpt": 0, "lcr": 0.163333333333333, "gpqa": 0.405050505050505, "omniscience": -60.18333333333333, "omniscienceAccuracy": 0.0815, "omniscienceNonHallucination": 0.2560333877699147, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "DeepSeek V3 0324", "shortName": "DeepSeek V3 0324", "slug": "deepseek-v3-0324", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 9.72000501684011, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.020199039576443884, "terminalBench40": 0, "terminalBench21": 0.138576779026217, "terminalbenchHard": 0.151515151515152, "scicode": 0.390046296296296, "hle": 0.0474, "critpt": 0, "lcr": 0.406666666666667, "gpqa": 0.654545454545455, "omniscience": -40.666666666666664, "omniscienceAccuracy": 0.24316666666666667, "omniscienceNonHallucination": 0.14137855097996033, "gdpvalNormalized": 0, "price1mInputTokens": 0.24, "price1mOutputTokens": 0.9, "cacheHitPrice": 0.135, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Command-R+ (Apr '24)", "shortName": "Command-R+ (Apr)", "slug": "command-r-plus-04-2024", "isReasoning": false, "deprecated": true, "modelCreatorName": "Cohere", "intelligenceIndex": 5.29816467823515, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0463, "critpt": null, "lcr": null, "gpqa": 0.323232323232323, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Sarvam 30B (High)", "shortName": "Sarvam 30B (high)", "slug": "sarvam-30b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Sarvam", "intelligenceIndex": 6.55971145066503, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0750695088044486, "critpt": 0.00285714285714286, "lcr": 0, "gpqa": 0.633333333333333, "omniscience": -71.51666666666667, "omniscienceAccuracy": 0.12616666666666668, "omniscienceNonHallucination": 0.03719244707228686, "gdpvalNormalized": null, "price1mInputTokens": 0.026, "price1mOutputTokens": 0.11, "cacheHitPrice": 0.016, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "Qwen2 Instruct 72B", "shortName": "Qwen2 72B", "slug": "qwen2-72b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.32869002003161, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0366, "critpt": null, "lcr": null, "gpqa": 0.370707070707071, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Mercury 2.5", "shortName": "Mercury 2.5", "slug": "mercury-2-5", "isReasoning": true, "deprecated": false, "modelCreatorName": "Inception", "intelligenceIndex": 12.3408453757638, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.11668472036336801, "terminalBench40": 0, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.385416666666667, "hle": 0.117701575532901, "critpt": 0, "lcr": 0.716666666666667, "gpqa": null, "omniscience": -39.483333333333334, "omniscienceAccuracy": 0.2265, "omniscienceNonHallucination": 0.19672484378366728, "gdpvalNormalized": 0, "price1mInputTokens": 0.25, "price1mOutputTokens": 0.75, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 260000, "medianOutputTokensPerSecond": 636.506822509039, "medianTimeToFirstTokenSeconds": 3.36430465700004, "isOpenWeights": false }, { "name": "GPT-4 Turbo", "shortName": "GPT-4 Turbo", "slug": "gpt-4-turbo", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 7.04351520157305, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0313, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 10, "price1mOutputTokens": 30, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 31.8297806668832, "medianTimeToFirstTokenSeconds": 3.40006902399989, "isOpenWeights": false }, { "name": "Nova 2.0 Lite (Non-reasoning)", "shortName": "Nova 2.0 Lite (non-reasoning)", "slug": "nova-2-0-lite", "isReasoning": false, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 8.73661367769335, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0287303058387396, "critpt": 0, "lcr": 0.186666666666667, "gpqa": 0.603030303030303, "omniscience": -59.81666666666667, "omniscienceAccuracy": 0.14016666666666666, "omniscienceNonHallucination": 0.1413064547392906, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 168.519336528832, "medianTimeToFirstTokenSeconds": 1.05692980849995, "isOpenWeights": false }, { "name": "NVIDIA Nemotron 3 Nano 30B A3B (Reasoning)", "shortName": "Nemotron 3 Nano", "slug": "nvidia-nemotron-3-nano-30b-a3b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 8.8961440122188, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.01675160944475661, "terminalBench40": 0, "terminalBench21": 0.0674157303370786, "terminalbenchHard": 0.136363636363636, "scicode": 0.305555555555556, "hle": 0.113994439295644, "critpt": 0.00857142857142857, "lcr": 0.38, "gpqa": 0.756565656565657, "omniscience": -51.56666666666667, "omniscienceAccuracy": 0.173, "omniscienceNonHallucination": 0.16727126158806938, "gdpvalNormalized": 0, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.2, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 196.967299746769, "medianTimeToFirstTokenSeconds": 1.04436939849999, "isOpenWeights": true }, { "name": "DeepSeek V4 Flash Vision (Max)", "shortName": "DeepSeek V4 Flash Vision (max)", "slug": "deepseek-v4-flash-vision", "isReasoning": true, "deprecated": false, "modelCreatorName": "DeepSeek", "intelligenceIndex": 34.8390628035969, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.314372044499279, "terminalBench40": 0.121212121212121, "terminalBench21": 0.741573033707865, "terminalbenchHard": null, "scicode": 0.496527777777778, "hle": 0.344763670064875, "critpt": 0.108571428571429, "lcr": 0.813333333333333, "gpqa": 0.913131313131313, "omniscience": -17.633333333333333, "omniscienceAccuracy": 0.38566666666666666, "omniscienceNonHallucination": 0.08518719479110148, "gdpvalNormalized": 0.5172100000000001, "price1mInputTokens": 0.44, "price1mOutputTokens": 1.32, "cacheHitPrice": 0.014, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 214.871776265428, "medianTimeToFirstTokenSeconds": 0.937208634000001, "isOpenWeights": false }, { "name": "DeepSeek LLM 67B Chat (V1)", "shortName": "DeepSeek LLM 67B (V1)", "slug": "deepseek-llm-67b-chat", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 5.3045047203477, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "MiniCPM5-1B (Reasoning)", "shortName": "MiniCPM5-1B", "slug": "minicpm5-1b", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenBMB", "intelligenceIndex": 8.79664321370212, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0648748841519926, "critpt": 0, "lcr": 0.06, "gpqa": 0.277777777777778, "omniscience": -14.616666666666667, "omniscienceAccuracy": 0.02033333333333333, "omniscienceNonHallucination": 0.8300442327322218, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-6 Sol (Medium)", "shortName": "GPT-6 Sol (medium)", "slug": "gpt-6-sol-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 39.7820971749522, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.24721927010382636, "terminalBench40": 0.186868686868687, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.538194444444444, "hle": 0.409638554216867, "critpt": 0.245714285714286, "lcr": 0.823333333333333, "gpqa": null, "omniscience": 27.016666666666666, "omniscienceAccuracy": 0.5345, "omniscienceNonHallucination": 0.43215180809165776, "gdpvalNormalized": 0.41005499999999995, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 872000, "isOpenWeights": false }, { "name": "Claude 3 Opus", "shortName": "Claude 3 Opus", "slug": "claude-3-opus", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 8.72397483423503, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0275, "critpt": null, "lcr": null, "gpqa": 0.488888888888889, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 15, "price1mOutputTokens": 75, "cacheHitPrice": 1.5, "cacheWritePrice": 18.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Gemini 3.6 Flash (High)", "shortName": "Gemini 3.6 Flash (high)", "slug": "gemini-3-6-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 33.9785928920714, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.9288448964661649, "terminalBench40": 0.0707070707070707, "terminalBench21": 0.775280898876405, "terminalbenchHard": null, "scicode": 0.533564814814815, "hle": 0.408248378127896, "critpt": 0.105714285714286, "lcr": 0.8, "gpqa": 0.928282828282828, "omniscience": 22.133333333333333, "omniscienceAccuracy": 0.49966666666666665, "omniscienceNonHallucination": 0.4437041972018654, "gdpvalNormalized": 0.38233500000000004, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 183.928736538462, "medianTimeToFirstTokenSeconds": 17.994500725, "isOpenWeights": false }, { "name": "Gemini 3 Flash Preview (Non-reasoning)", "shortName": "Gemini 3 Flash (non-reasoning)", "slug": "gemini-3-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 17.9336848163786, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.318181818181818, "scicode": null, "hle": 0.14967562557924, "critpt": 0.0142857142857143, "lcr": 0.553333333333333, "gpqa": 0.812121212121212, "omniscience": -4.316666666666666, "omniscienceAccuracy": 0.4578333333333333, "omniscienceNonHallucination": 0.07592991085152168, "gdpvalNormalized": null, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 172.173470398116, "medianTimeToFirstTokenSeconds": 1.09317152449998, "isOpenWeights": false }, { "name": "Jamba 1.7 Large", "shortName": "Jamba 1.7 Large", "slug": "jamba-1-7-large", "isReasoning": false, "deprecated": false, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 6.0710365237197, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0366079703429101, "critpt": 0, "lcr": 0.193333333333333, "gpqa": 0.38989898989899, "omniscience": -54.916666666666664, "omniscienceAccuracy": 0.20583333333333334, "omniscienceNonHallucination": 0.04931794333683104, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Grok 4.5 (High)", "shortName": "Grok 4.5 (high)", "slug": "grok-4-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 38.8121153240657, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.0364552526545159, "terminalBench40": 0.106060606060606, "terminalBench21": 0.816479400749064, "terminalbenchHard": null, "scicode": 0.549768518518518, "hle": 0.42678405931418, "critpt": 0.154285714285714, "lcr": 0.793333333333333, "gpqa": 0.931313131313131, "omniscience": 25.316666666666666, "omniscienceAccuracy": 0.5155, "omniscienceNonHallucination": 0.45854833161334707, "gdpvalNormalized": 0.43522000000000005, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.3, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 57.2000013332608, "medianTimeToFirstTokenSeconds": 10.223569894, "isOpenWeights": false }, { "name": "Mistral Saba", "shortName": "Mistral Saba", "slug": "mistral-saba", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 6.48598593446849, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.043, "critpt": null, "lcr": null, "gpqa": 0.424242424242424, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": false }, { "name": "Qwen3 Max", "shortName": "Qwen3 Max", "slug": "qwen3-max", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 15.6098449457502, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.204545454545455, "scicode": null, "hle": 0.119091751621872, "critpt": 0, "lcr": 0.5, "gpqa": 0.763636363636364, "omniscience": -43.55, "omniscienceAccuracy": 0.24416666666666667, "omniscienceNonHallucination": 0.1007717750826902, "gdpvalNormalized": null, "price1mInputTokens": 1.2, "price1mOutputTokens": 6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 56.6056425016427, "medianTimeToFirstTokenSeconds": 2.20049802450001, "isOpenWeights": false }, { "name": "Gemini 2.0 Flash-Lite (Feb '25)", "shortName": "Gemini 2.0 Flash-Lite (Feb)", "slug": "gemini-2-0-flash-lite-001", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 7.40799139518293, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0335, "critpt": null, "lcr": null, "gpqa": 0.535353535353535, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Mistral Small (Sep '24)", "shortName": "Mistral Small (Sep)", "slug": "mistral-small", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.84842869297048, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0425, "critpt": null, "lcr": null, "gpqa": 0.380808080808081, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "GPT-5.6 Sol (Non-reasoning)", "shortName": "GPT-5.6 Sol (non-reasoning)", "slug": "gpt-5-6-sol-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 28.3280042499237, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.741573033707865, "terminalbenchHard": null, "scicode": 0.476851851851852, "hle": 0.166821130676552, "critpt": 0.0514285714285714, "lcr": 0.623333333333333, "gpqa": 0.78989898989899, "omniscience": 1.1, "omniscienceAccuracy": 0.487, "omniscienceNonHallucination": 0.0721247563352827, "gdpvalNormalized": 0.36288, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.4, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 66.299136593131, "medianTimeToFirstTokenSeconds": 1.03367705300002, "isOpenWeights": false }, { "name": "Gemini 3 Flash Preview (Reasoning)", "shortName": "Gemini 3 Flash", "slug": "gemini-3-flash-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 26.3272067856089, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.386363636363636, "scicode": null, "hle": 0.365616311399444, "critpt": 0.0857142857142857, "lcr": 0.78, "gpqa": 0.897979797979798, "omniscience": 10.133333333333333, "omniscienceAccuracy": 0.5343333333333333, "omniscienceNonHallucination": 0.0701503221188261, "gdpvalNormalized": null, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 186.116074189644, "medianTimeToFirstTokenSeconds": 7.17180999299995, "isOpenWeights": false }, { "name": "GPT-6 Luna (High)", "shortName": "GPT-6 Luna (high)", "slug": "gpt-6-luna-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 32.1482304150837, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.029067566599182932, "terminalBench40": 0.0454545454545455, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.503472222222222, "hle": 0.329471733086191, "critpt": 0.154285714285714, "lcr": 0.793333333333333, "gpqa": null, "omniscience": -5.5, "omniscienceAccuracy": 0.428, "omniscienceNonHallucination": 0.15559440559440563, "gdpvalNormalized": 0.39518499999999995, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.01, "cacheWritePrice": 0.125, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 122.95260169383, "medianTimeToFirstTokenSeconds": 11.9969286605001, "isOpenWeights": false }, { "name": "Nanbeige4.1-3B", "shortName": "Nanbeige4.1-3B", "slug": "nanbeige4-1-3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Nanbeige", "intelligenceIndex": 8.4012282586083, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0112359550561798, "terminalbenchHard": 0, "scicode": null, "hle": 0.108897126969416, "critpt": 0, "lcr": 0, "gpqa": 0.84949494949495, "omniscience": -42.5, "omniscienceAccuracy": 0.09866666666666667, "omniscienceNonHallucination": 0.41900887573964496, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Qwen3.8 27B (Low)", "shortName": "Qwen3.8 27B (low)", "slug": "qwen3-8-27b-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 26.2047978189896, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.0482225365160391, "terminalBench40": 0.0252525252525253, "terminalBench21": 0.674157303370786, "terminalbenchHard": null, "scicode": 0.400462962962963, "hle": 0.140407784986098, "critpt": 0, "lcr": 0.773333333333333, "gpqa": 0.845454545454545, "omniscience": -26.666666666666668, "omniscienceAccuracy": 0.17133333333333334, "omniscienceNonHallucination": 0.4714400643604183, "gdpvalNormalized": 0.42078499999999996, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.1, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 49.6305191556248, "medianTimeToFirstTokenSeconds": 3.84855280299996, "isOpenWeights": true }, { "name": "Solar Open2 250B", "shortName": "Solar Open2 250B", "slug": "solar-open2-250b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Upstage", "intelligenceIndex": 24.7365297627523, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.441947565543071, "terminalbenchHard": null, "scicode": 0.480324074074074, "hle": 0.285449490268767, "critpt": 0.0571428571428571, "lcr": 0.716666666666667, "gpqa": 0.856565656565657, "omniscience": -1.7666666666666666, "omniscienceAccuracy": 0.18833333333333332, "omniscienceNonHallucination": 0.7462012320328542, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1048576, "isOpenWeights": true }, { "name": "Ling 3.0 Tiny", "shortName": "Ling 3.0 Tiny", "slug": "ling-3-0-tiny", "isReasoning": true, "deprecated": false, "modelCreatorName": "InclusionAI", "intelligenceIndex": 11.0636233769305, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0, "terminalBench40": 0, "terminalBench21": 0.277153558052434, "terminalbenchHard": null, "scicode": 0.241898148148148, "hle": 0.0931417979610751, "critpt": 0, "lcr": 0.603333333333333, "gpqa": 0.734343434343434, "omniscience": -19.35, "omniscienceAccuracy": 0.08516666666666667, "omniscienceNonHallucination": 0.6953907815631262, "gdpvalNormalized": 0.027389999999999987, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 56.6918610426526, "medianTimeToFirstTokenSeconds": 2.647066717, "isOpenWeights": true }, { "name": "Qwen3 30B A3B 2507 Instruct", "shortName": "Qwen3 30B A3B 2507 (non-reasoning)", "slug": "qwen3-30b-a3b-2507", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.52105693426478, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0690454124189064, "critpt": 0, "lcr": 0.263333333333333, "gpqa": 0.658585858585859, "omniscience": -66.15, "omniscienceAccuracy": 0.14966666666666667, "omniscienceNonHallucination": 0.04606036848294781, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 141.560650678683, "medianTimeToFirstTokenSeconds": 1.87899648049995, "isOpenWeights": true }, { "name": "GPT-5 (Low)", "shortName": "GPT-5 (low)", "slug": "gpt-5-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.7855980255103, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.265151515151515, "scicode": null, "hle": 0.195551436515292, "critpt": 0.0114285714285714, "lcr": null, "gpqa": 0.808080808080808, "omniscience": -10.783333333333333, "omniscienceAccuracy": 0.38116666666666665, "omniscienceNonHallucination": 0.20980339348235932, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 69.984254723245, "medianTimeToFirstTokenSeconds": 7.885768171, "isOpenWeights": false }, { "name": "Celeris-1", "shortName": "Celeris-1", "slug": "celeris-1", "isReasoning": false, "deprecated": false, "modelCreatorName": "Celeris", "intelligenceIndex": 6.3469533538601, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.05015755586591194, "terminalBench40": 0, "terminalBench21": 0.112359550561798, "terminalbenchHard": null, "scicode": 0.216435185185185, "hle": 0.0681186283595922, "critpt": 0, "lcr": 0.38, "gpqa": 0.631313131313131, "omniscience": -71.58333333333333, "omniscienceAccuracy": 0.10983333333333334, "omniscienceNonHallucination": 0.07245834113461902, "gdpvalNormalized": 0, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.7, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 1535.25129347095, "medianTimeToFirstTokenSeconds": 0.563656820999995, "isOpenWeights": false }, { "name": "Qwen3 Max Thinking", "shortName": "Qwen3 Max Thinking", "slug": "qwen3-max-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 21.2551111296741, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.242424242424242, "scicode": null, "hle": 0.279888785912882, "critpt": 0.0171428571428571, "lcr": 0.743333333333333, "gpqa": 0.860606060606061, "omniscience": -36.61666666666667, "omniscienceAccuracy": 0.2981666666666667, "omniscienceNonHallucination": 0.05343148895749228, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Llama 3.2 Instruct 1B", "shortName": "Llama 3.2 1B", "slug": "llama-3-2-instruct-1b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0546, "critpt": 0, "lcr": 0.0666666666666667, "gpqa": 0.195959595959596, "omniscience": -54.7, "omniscienceAccuracy": 0.07, "omniscienceNonHallucination": 0.3365591397849462, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Tiny Aya Global", "shortName": "Tiny Aya Global", "slug": "tiny-aya-global", "isReasoning": false, "deprecated": false, "modelCreatorName": "Cohere", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0523632993512512, "critpt": 0, "lcr": 0, "gpqa": 0.305050505050505, "omniscience": -84.3, "omniscienceAccuracy": 0.06066666666666667, "omniscienceNonHallucination": 0.03797019162526616, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "GPT-5.6 Terra (High)", "shortName": "GPT-5.6 Terra (high)", "slug": "gpt-5-6-terra-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 34.2376558067555, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.3378727947949101, "terminalBench40": 0.0151515151515152, "terminalBench21": 0.756554307116105, "terminalbenchHard": 0.575757575757576, "scicode": 0.524305555555556, "hle": 0.385078776645042, "critpt": 0.228571428571429, "lcr": 0.776666666666667, "gpqa": 0.895959595959596, "omniscience": -3.466666666666667, "omniscienceAccuracy": 0.4548333333333333, "omniscienceNonHallucination": 0.10210944665239985, "gdpvalNormalized": 0.43127499999999996, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 72.3499538446266, "medianTimeToFirstTokenSeconds": 2.95910261099995, "isOpenWeights": false }, { "name": "GLM-4.7 (Non-reasoning)", "shortName": "GLM-4.7 (non-reasoning)", "slug": "glm-4-7-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 17.3564299133682, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.303030303030303, "scicode": null, "hle": 0.0639481000926784, "critpt": 0, "lcr": 0.406666666666667, "gpqa": 0.663636363636364, "omniscience": -47.31666666666667, "omniscienceAccuracy": 0.23633333333333334, "omniscienceNonHallucination": 0.07092972501091221, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 2.2, "cacheHitPrice": 0.28500000000000003, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 60.8163582878094, "medianTimeToFirstTokenSeconds": 1.29554015500003, "isOpenWeights": true }, { "name": "A.X-K2", "shortName": "A.X-K2", "slug": "a-x-k2", "isReasoning": true, "deprecated": false, "modelCreatorName": "SK Telecom", "intelligenceIndex": 22.7039007610661, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.389513108614232, "terminalbenchHard": null, "scicode": 0.409722222222222, "hle": 0.295644114921223, "critpt": 0.0885714285714286, "lcr": 0.7, "gpqa": 0.856565656565657, "omniscience": -8.233333333333333, "omniscienceAccuracy": 0.18616666666666667, "omniscienceNonHallucination": 0.670079868933033, "gdpvalNormalized": 0.22676, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Exaone 4.0 1.2B (Non-reasoning)", "shortName": "Exaone 4.0 1.2B (non-reasoning)", "slug": "exaone-4-0-1-2b", "isReasoning": false, "deprecated": false, "modelCreatorName": "LG AI Research", "intelligenceIndex": 5.22938457197929, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0569972196478221, "critpt": 0, "lcr": 0, "gpqa": 0.424242424242424, "omniscience": -82.15, "omniscienceAccuracy": 0.05, "omniscienceNonHallucination": 0.08263157894736839, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 64000, "isOpenWeights": true }, { "name": "Llama 3 Instruct 8B", "shortName": "Llama 3 8B", "slug": "llama-3-instruct-8b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0506, "critpt": 0, "lcr": 0, "gpqa": 0.295959595959596, "omniscience": -70.11666666666666, "omniscienceAccuracy": 0.10366666666666667, "omniscienceNonHallucination": 0.1020825585719598, "gdpvalNormalized": null, "price1mInputTokens": 0.045, "price1mOutputTokens": 0.145, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "Hermes 4 - Llama-3.1 405B (Reasoning)", "shortName": "Hermes 4 405B", "slug": "hermes-4-llama-3-1-405b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Nous Research", "intelligenceIndex": 7.49497809925135, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.113636363636364, "scicode": null, "hle": 0.108897126969416, "critpt": 0.00285714285714286, "lcr": 0.223333333333333, "gpqa": 0.727272727272727, "omniscience": -36.016666666666666, "omniscienceAccuracy": 0.30083333333333334, "omniscienceNonHallucination": 0.054588796185935595, "gdpvalNormalized": null, "price1mInputTokens": 1, "price1mOutputTokens": 3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 38.4633711405253, "medianTimeToFirstTokenSeconds": 2.40825708499995, "isOpenWeights": true }, { "name": "MiMo-V2-Flash (Non-reasoning)", "shortName": "MiMo-V2-Flash (non-reasoning)", "slug": "mimo-v2-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 16.0498614373966, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.617977528089888, "terminalbenchHard": 0.257575757575758, "scicode": null, "hle": 0.0857275254865616, "critpt": 0, "lcr": 0.363333333333333, "gpqa": 0.655555555555556, "omniscience": -48.45, "omniscienceAccuracy": 0.15633333333333332, "omniscienceNonHallucination": 0.2404188067957329, "gdpvalNormalized": 0.05725999999999999, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Granite 4.1 8B", "shortName": "Granite 4.1 8B", "slug": "granite-4-1-8b", "isReasoning": false, "deprecated": true, "modelCreatorName": "IBM", "intelligenceIndex": 6.57208916161427, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0337078651685393, "terminalbenchHard": 0, "scicode": null, "hle": 0.0379981464318814, "critpt": 0, "lcr": 0.133333333333333, "gpqa": 0.433333333333333, "omniscience": -64.11666666666666, "omniscienceAccuracy": 0.123, "omniscienceNonHallucination": 0.12865830482706198, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "DeepSeek V4 Pro 0813 (Max)", "shortName": "DeepSeek V4 Pro 0813 (max)", "slug": "deepseek-v4-pro", "isReasoning": true, "deprecated": false, "modelCreatorName": "DeepSeek", "intelligenceIndex": 35.9967791278402, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.6740219040435134, "terminalBench40": 0.141414141414141, "terminalBench21": 0.786516853932584, "terminalbenchHard": null, "scicode": 0.510416666666667, "hle": 0.410101946246525, "critpt": 0.18, "lcr": 0.803333333333333, "gpqa": 0.928282828282828, "omniscience": 0.8333333333333334, "omniscienceAccuracy": 0.491, "omniscienceNonHallucination": 0.05173542894564509, "gdpvalNormalized": 0.47071, "price1mInputTokens": 1.32, "price1mOutputTokens": 3.96, "cacheHitPrice": 0.044, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 81.0330261039759, "medianTimeToFirstTokenSeconds": 1.669593397, "isOpenWeights": true }, { "name": "Claude Instant", "shortName": "Claude Instant", "slug": "claude-instant", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 5.03498213171439, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0353, "critpt": null, "lcr": null, "gpqa": 0.33030303030303, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 100000, "isOpenWeights": false }, { "name": "Gemini 2.5 Flash-Lite Preview (Sep '25) (Reasoning)", "shortName": "Gemini 2.5 Flash-Lite (Sep)", "slug": "gemini-2-5-flash-lite-preview-09-2025-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 10.3543486383322, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.128787878787879, "scicode": null, "hle": 0.0695088044485635, "critpt": 0, "lcr": 0.646666666666667, "gpqa": 0.709090909090909, "omniscience": -54.016666666666666, "omniscienceAccuracy": 0.17983333333333335, "omniscienceNonHallucination": 0.12212964844543794, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "o4-mini (High)", "shortName": "o4-mini (high)", "slug": "o4-mini", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 16.653105214846, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.151515151515152, "scicode": null, "hle": 0.165430954587581, "critpt": 0.00571428571428571, "lcr": 0.61, "gpqa": 0.783838383838384, "omniscience": -35.7, "omniscienceAccuracy": 0.248, "omniscienceNonHallucination": 0.19547872340425532, "gdpvalNormalized": null, "price1mInputTokens": 1.1, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.275, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 138.903073210703, "medianTimeToFirstTokenSeconds": 32.5066079795, "isOpenWeights": false }, { "name": "GPT-5.2 (Medium)", "shortName": "GPT-5.2 (medium)", "slug": "gpt-5-2-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 26.5009561703275, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.431818181818182, "scicode": null, "hle": 0.266913809082484, "critpt": 0.0785714285714285, "lcr": 0.703333333333333, "gpqa": 0.863636363636364, "omniscience": 0.26666666666666666, "omniscienceAccuracy": 0.38283333333333336, "omniscienceNonHallucination": 0.384012962462868, "gdpvalNormalized": null, "price1mInputTokens": 1.75, "price1mOutputTokens": 14, "cacheHitPrice": 0.175, "cacheWritePrice": null, "contextWindowTokens": 400000, "isOpenWeights": false }, { "name": "Magistral Medium 1.2", "shortName": "Magistral Medium 1.2", "slug": "magistral-medium-2509", "isReasoning": true, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 11.8254509953707, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.123595505617978, "terminalbenchHard": 0.128787878787879, "scicode": null, "hle": 0.102873030583874, "critpt": 0.00285714285714286, "lcr": 0.53, "gpqa": 0.739393939393939, "omniscience": -26.733333333333334, "omniscienceAccuracy": 0.20783333333333334, "omniscienceNonHallucination": 0.40016831474857983, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Qwen3.6 35B A3B (Reasoning)", "shortName": "Qwen3.6 35B A3B", "slug": "qwen3-6-35b-a3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 18.2289900189843, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.4750534562511306, "terminalBench40": 0, "terminalBench21": 0.449438202247191, "terminalbenchHard": 0.348484848484849, "scicode": 0.365740740740741, "hle": 0.222428174235403, "critpt": 0.00285714285714286, "lcr": 0.716666666666667, "gpqa": 0.841414141414142, "omniscience": -22.183333333333334, "omniscienceAccuracy": 0.18816666666666668, "omniscienceNonHallucination": 0.4949702319852186, "gdpvalNormalized": 0.18960500000000002, "price1mInputTokens": 0.375, "price1mOutputTokens": 2.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 125.260964594678, "medianTimeToFirstTokenSeconds": 2.09584000550004, "isOpenWeights": true }, { "name": "GPT-5.6 Luna (Xhigh)", "shortName": "GPT-5.6 Luna (xhigh)", "slug": "gpt-5-6-luna-xhigh", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 34.555348822309, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.085342806135728, "terminalBench40": 0.0353535353535354, "terminalBench21": 0.779026217228464, "terminalbenchHard": null, "scicode": 0.50462962962963, "hle": 0.369786839666358, "critpt": 0.205714285714286, "lcr": 0.816666666666667, "gpqa": 0.894949494949495, "omniscience": -10.766666666666667, "omniscienceAccuracy": 0.4245, "omniscienceNonHallucination": 0.07529684332464526, "gdpvalNormalized": 0.441855, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.02, "cacheWritePrice": 0.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 111.311476045069, "medianTimeToFirstTokenSeconds": 47.222149855, "isOpenWeights": false }, { "name": "Gemini 2.5 Flash Preview (Sep '25) (Non-reasoning)", "shortName": "Gemini 2.5 Flash (Sep) (non-reasoning)", "slug": "gemini-2-5-flash-preview-09-2025", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 12.3823294842565, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.143939393939394, "scicode": null, "hle": 0.0871177015755329, "critpt": 0, "lcr": 0.6, "gpqa": 0.765656565656566, "omniscience": -39.9, "omniscienceAccuracy": 0.2683333333333333, "omniscienceNonHallucination": 0.08792710706150342, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "QwQ 32B-Preview", "shortName": "QwQ 32B-Preview", "slug": "QwQ-32B-Preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.58699681312465, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0391, "critpt": null, "lcr": null, "gpqa": 0.556565656565656, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Ling-flash-2.0", "shortName": "Ling-flash-2.0", "slug": "ling-flash-2-0", "isReasoning": false, "deprecated": true, "modelCreatorName": "InclusionAI", "intelligenceIndex": 7.8074626926252, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.106060606060606, "scicode": null, "hle": 0.06209453197405, "critpt": 0, "lcr": 0.176666666666667, "gpqa": 0.656565656565657, "omniscience": -62.7, "omniscienceAccuracy": 0.14133333333333334, "omniscienceNonHallucination": 0.10520186335403725, "gdpvalNormalized": null, "price1mInputTokens": 0.14, "price1mOutputTokens": 0.57, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 5.05154803104264, "medianTimeToFirstTokenSeconds": 2.49916217300003, "isOpenWeights": true }, { "name": "GPT-5.4 nano (Medium)", "shortName": "GPT-5.4 nano (medium)", "slug": "gpt-5-4-nano-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.0139276661357, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.333333333333333, "scicode": null, "hle": 0.159406858202039, "critpt": 0.0514285714285714, "lcr": 0.673333333333333, "gpqa": 0.760606060606061, "omniscience": -18.033333333333335, "omniscienceAccuracy": 0.219, "omniscienceNonHallucination": 0.4886897140418267, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.25, "cacheHitPrice": 0.02, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 168.684197403289, "medianTimeToFirstTokenSeconds": 5.45082188000003, "isOpenWeights": false }, { "name": "Apertus 8B Instruct", "shortName": "Apertus 8B Instruct", "slug": "apertus-8b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Swiss AI Initiative", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0495829471733086, "critpt": 0, "lcr": 0, "gpqa": 0.255555555555556, "omniscience": -77.21666666666667, "omniscienceAccuracy": 0.0965, "omniscienceNonHallucination": 0.03855377236672197, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "DeepSeek V4 Flash 0420 (High)", "shortName": "DeepSeek V4 Flash (high)", "slug": "deepseek-v4-flash-0420-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 24.3839569152521, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.12711665874155298, "terminalBench40": 0.0303030303030303, "terminalBench21": 0.569288389513109, "terminalbenchHard": 0.386363636363636, "scicode": 0.40162037037037, "hle": 0.30259499536608, "critpt": 0.0342857142857143, "lcr": 0.72, "gpqa": 0.866666666666667, "omniscience": -23.083333333333332, "omniscienceAccuracy": 0.3486666666666667, "omniscienceNonHallucination": 0.11028659160696008, "gdpvalNormalized": 0.24734500000000004, "price1mInputTokens": 0.1119, "price1mOutputTokens": 0.23750000000000002, "cacheHitPrice": 0.028, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "DeepSeek-V2.5", "shortName": "DeepSeek-V2.5", "slug": "deepseek-v2-5-sep-2024", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 6.55308603019382, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Claude 4 Opus (Reasoning)", "shortName": "Claude 4 Opus", "slug": "claude-4-opus-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 20.6446355102762, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.310606060606061, "scicode": null, "hle": 0.123262279888786, "critpt": null, "lcr": 0.693333333333333, "gpqa": 0.795959595959596, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 15, "price1mOutputTokens": 75, "cacheHitPrice": 1.5, "cacheWritePrice": 18.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Gemini 3.8 Flash (High)", "shortName": "Gemini 3.8 Flash (high)", "slug": "gemini-3-8-flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 40.9262321765904, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.2427947606950427, "terminalBench40": 0.196969696969697, "terminalBench21": 0.876404494382023, "terminalbenchHard": null, "scicode": 0.565972222222222, "hle": 0.478220574606117, "critpt": 0.182857142857143, "lcr": 0.813333333333333, "gpqa": 0.952525252525253, "omniscience": 29.55, "omniscienceAccuracy": 0.546, "omniscienceNonHallucination": 0.44823788546255505, "gdpvalNormalized": 0.455975, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.075, "cacheWritePrice": 0.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 235.516384825237, "medianTimeToFirstTokenSeconds": 21.762273558, "isOpenWeights": false }, { "name": "GPT-5.6 Sol (High)", "shortName": "GPT-5.6 Sol (high)", "slug": "gpt-5-6-sol-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 42.3461010420519, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.807982690134498, "terminalBench40": 0.207070707070707, "terminalBench21": 0.872659176029963, "terminalbenchHard": 0.621212121212121, "scicode": 0.577546296296296, "hle": 0.46014828544949, "critpt": 0.257142857142857, "lcr": 0.816666666666667, "gpqa": 0.928282828282828, "omniscience": 20.366666666666667, "omniscienceAccuracy": 0.5835, "omniscienceNonHallucination": 0.0880352140856343, "gdpvalNormalized": 0.490015, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.4, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 78.2773880153449, "medianTimeToFirstTokenSeconds": 11.8036558075, "isOpenWeights": false }, { "name": "GPT-4o (Aug '24)", "shortName": "GPT-4o (Aug)", "slug": "gpt-4o-2024-08-06", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 7.73970909995321, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0233, "critpt": 0, "lcr": 0.41, "gpqa": 0.521212121212121, "omniscience": -20.866666666666667, "omniscienceAccuracy": 0.237, "omniscienceNonHallucination": 0.41590214067278286, "gdpvalNormalized": null, "price1mInputTokens": 2.5, "price1mOutputTokens": 10, "cacheHitPrice": 1.25, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 95.7378896084168, "medianTimeToFirstTokenSeconds": 1.18729517650002, "isOpenWeights": false }, { "name": "Qwen3.6 27B (Reasoning)", "shortName": "Qwen3.6 27B", "slug": "qwen3-6-27b", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 21.4279902240782, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.621236893699357, "terminalBench40": 0, "terminalBench21": 0.606741573033708, "terminalbenchHard": 0.348484848484849, "scicode": 0.428240740740741, "hle": 0.230769230769231, "critpt": 0.0114285714285714, "lcr": 0.773333333333333, "gpqa": 0.842424242424242, "omniscience": -20.016666666666666, "omniscienceAccuracy": 0.196, "omniscienceNonHallucination": 0.5072553897180763, "gdpvalNormalized": 0.23667000000000002, "price1mInputTokens": 0.6, "price1mOutputTokens": 3.6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 56.1610411868473, "medianTimeToFirstTokenSeconds": 3.66738588899997, "isOpenWeights": true }, { "name": "Nemotron 3 Nano Omni 30B A3B Reasoning", "shortName": "Nemotron 3 Nano Omni 30B A3B", "slug": "nemotron-3-nano-omni-30b-a3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 10.2506761710987, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0674157303370786, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0481927710843374, "critpt": 0, "lcr": 0.396666666666667, "gpqa": 0.468686868686869, "omniscience": -57.43333333333333, "omniscienceAccuracy": 0.152, "omniscienceNonHallucination": 0.1434748427672956, "gdpvalNormalized": 0, "price1mInputTokens": 0.195, "price1mOutputTokens": 1.095, "cacheHitPrice": 0.195, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 263.67388270435, "medianTimeToFirstTokenSeconds": 0.389989410999988, "isOpenWeights": true }, { "name": "Hy3-preview (Reasoning)", "shortName": "Hy3-preview", "slug": "hy3-preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "Tencent", "intelligenceIndex": 22.7362028668659, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.340909090909091, "scicode": null, "hle": 0.277571825764597, "critpt": 0.0457142857142857, "lcr": 0.646666666666667, "gpqa": 0.866666666666667, "omniscience": -35.05, "omniscienceAccuracy": 0.2793333333333333, "omniscienceNonHallucination": 0.12604070305272896, "gdpvalNormalized": null, "price1mInputTokens": 0.063, "price1mOutputTokens": 0.21, "cacheHitPrice": 0.021, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Kimi K2.7 Code", "shortName": "Kimi K2.7 Code", "slug": "kimi-k2-7-code", "isReasoning": true, "deprecated": false, "modelCreatorName": "Kimi", "intelligenceIndex": 25.8121062401836, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.541190247118529, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.674157303370786, "terminalbenchHard": 0.446969696969697, "scicode": 0.478009259259259, "hle": 0.35032437442076, "critpt": 0.1, "lcr": 0.793333333333333, "gpqa": 0.895959595959596, "omniscience": -10.216666666666667, "omniscienceAccuracy": 0.39566666666666667, "omniscienceNonHallucination": 0.1762272476558191, "gdpvalNormalized": 0.26252, "price1mInputTokens": 0.95, "price1mOutputTokens": 4, "cacheHitPrice": 0.19, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 82.8548703952724, "medianTimeToFirstTokenSeconds": 2.94720539149998, "isOpenWeights": true }, { "name": "Gemini 1.0 Pro", "shortName": "Gemini 1.0 Pro", "slug": "gemini-1-0-pro", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 5.34239163352755, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0419, "critpt": null, "lcr": null, "gpqa": 0.276767676767677, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": false }, { "name": "Qwen3.8 2.4T A95B", "shortName": "Qwen3.8 2.4T A95B", "slug": "qwen3-8-2-4t-a95b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 39.8861723105325, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.1559170502434677, "terminalBench40": 0.111111111111111, "terminalBench21": 0.820224719101124, "terminalbenchHard": null, "scicode": 0.540509259259259, "hle": 0.424467099165894, "critpt": 0.2, "lcr": 0.803333333333333, "gpqa": 0.935353535353535, "omniscience": 4.316666666666666, "omniscienceAccuracy": 0.3125, "omniscienceNonHallucination": 0.6082424242424243, "gdpvalNormalized": 0.548635, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.25, "cacheWritePrice": null, "contextWindowTokens": 262000, "medianOutputTokensPerSecond": 39.6155719859047, "medianTimeToFirstTokenSeconds": 2.89641871000003, "isOpenWeights": true }, { "name": "Solar Pro 2 (Preview) (Reasoning)", "shortName": "Solar Pro 2 (Preview)", "slug": "solar-pro-2-preview-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Upstage", "intelligenceIndex": 9.06998982069512, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0607043558850788, "critpt": null, "lcr": null, "gpqa": 0.577777777777778, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 64000, "isOpenWeights": false }, { "name": "GPT-5 nano (Medium)", "shortName": "GPT-5 nano (medium)", "slug": "gpt-5-nano-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 12.4802556704436, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.174242424242424, "scicode": null, "hle": 0.0871177015755329, "critpt": 0, "lcr": 0.436666666666667, "gpqa": 0.66969696969697, "omniscience": -25.783333333333335, "omniscienceAccuracy": 0.17583333333333334, "omniscienceNonHallucination": 0.4738119312436805, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.005, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 146.14094430852, "medianTimeToFirstTokenSeconds": 46.70211426, "isOpenWeights": false }, { "name": "DeepSeek-Coder-V2", "shortName": "DeepSeek-Coder-V2", "slug": "deepseek-coder-v2", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 5.98428477265092, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Olmo 3 7B Instruct", "shortName": "Olmo 3 7B", "slug": "olmo-3-7b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 5.24099718758815, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0579240037071362, "critpt": 0, "lcr": 0, "gpqa": 0.4, "omniscience": -77.06666666666666, "omniscienceAccuracy": 0.074, "omniscienceNonHallucination": 0.08783297336213103, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "JT-4.1 Flash 236B A21B (Non-reasoning)", "shortName": "JT-4.1 Flash 236B A21B (non-reasoning)", "slug": "jt-4-1-flash-236b-a21b", "isReasoning": false, "deprecated": false, "modelCreatorName": "China Mobile", "intelligenceIndex": 27.2926268381758, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.595505617977528, "terminalbenchHard": null, "scicode": null, "hle": 0.177942539388323, "critpt": 0, "lcr": 0.713333333333333, "gpqa": 0.845454545454545, "omniscience": -8.783333333333333, "omniscienceAccuracy": 0.24283333333333335, "omniscienceNonHallucination": 0.563284173453665, "gdpvalNormalized": 0.30716999999999994, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Grok 4.3 (Non-reasoning)", "shortName": "Grok 4.3 (non-reasoning)", "slug": "grok-4-3-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 13.9926809505055, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.16974590796834474, "terminalBench40": 0, "terminalBench21": 0.340823970037453, "terminalbenchHard": 0.189393939393939, "scicode": 0.393518518518519, "hle": 0.0676552363299351, "critpt": 0, "lcr": 0.323333333333333, "gpqa": 0.657575757575758, "omniscience": -33.083333333333336, "omniscienceAccuracy": 0.23716666666666666, "omniscienceNonHallucination": 0.2554074721433254, "gdpvalNormalized": 0.214555, "price1mInputTokens": 1.25, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.2, "cacheWritePrice": 1.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 105.671628730356, "medianTimeToFirstTokenSeconds": 0.782528865499998, "isOpenWeights": false }, { "name": "Claude 4.5 Sonnet (Reasoning)", "shortName": "Claude 4.5 Sonnet", "slug": "claude-4-5-sonnet-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 20.6661943537112, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5565810659988739, "terminalBench40": 0, "terminalBench21": 0.558052434456929, "terminalbenchHard": 0.356060606060606, "scicode": 0.457175925925926, "hle": 0.177942539388323, "critpt": 0.0114285714285714, "lcr": 0.723333333333333, "gpqa": 0.834343434343434, "omniscience": -0.08333333333333333, "omniscienceAccuracy": 0.32866666666666666, "omniscienceNonHallucination": 0.5091857000993049, "gdpvalNormalized": 0.196435, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 40.3715767098796, "medianTimeToFirstTokenSeconds": 13.2575140125, "isOpenWeights": false }, { "name": "Claude 4.5 Sonnet (Non-reasoning)", "shortName": "Claude 4.5 Sonnet (non-reasoning)", "slug": "claude-4-5-sonnet", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 19.3436534007474, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.287878787878788, "scicode": null, "hle": 0.072289156626506, "critpt": 0, "lcr": 0.54, "gpqa": 0.727272727272727, "omniscience": -9.366666666666667, "omniscienceAccuracy": 0.2838333333333333, "omniscienceNonHallucination": 0.4728880614382127, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 38.8138906100441, "medianTimeToFirstTokenSeconds": 1.1990986845, "isOpenWeights": false }, { "name": "Granite 4.1 3B", "shortName": "Granite 4.1 3B", "slug": "granite-4-1-3b", "isReasoning": false, "deprecated": true, "modelCreatorName": "IBM", "intelligenceIndex": 5.86186404175178, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0112359550561798, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0342910101946247, "critpt": 0, "lcr": 0.06, "gpqa": 0.314141414141414, "omniscience": -77.03333333333333, "omniscienceAccuracy": 0.0925, "omniscienceNonHallucination": 0.04921946740128558, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Ling 2.6 Flash", "shortName": "Ling 2.6 Flash", "slug": "ling-2-6-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "InclusionAI", "intelligenceIndex": 9.67895987024967, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.243445692883895, "terminalbenchHard": 0.212121212121212, "scicode": null, "hle": 0.0625579240037071, "critpt": 0, "lcr": 0.313333333333333, "gpqa": 0.592929292929293, "omniscience": -66.13333333333334, "omniscienceAccuracy": 0.1555, "omniscienceNonHallucination": 0.032761002565620645, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Llama 4 Maverick", "shortName": "Llama 4 Maverick", "slug": "llama-4-maverick", "isReasoning": false, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 9.9905959756501, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.0786516853932584, "terminalbenchHard": 0.0681818181818182, "scicode": 0.31712962962963, "hle": 0.0491195551436515, "critpt": 0, "lcr": 0.5, "gpqa": 0.670707070707071, "omniscience": -41.85, "omniscienceAccuracy": 0.249, "omniscienceNonHallucination": 0.11118508655126502, "gdpvalNormalized": 0, "price1mInputTokens": 0.26, "price1mOutputTokens": 0.9099999999999999, "cacheHitPrice": 0.245, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 84.32117672383, "medianTimeToFirstTokenSeconds": 0.954089652999983, "isOpenWeights": true }, { "name": "Jamba 1.6 Large", "shortName": "Jamba 1.6 Large", "slug": "jamba-1-6-large", "isReasoning": false, "deprecated": true, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 5.96740520286191, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0361, "critpt": null, "lcr": null, "gpqa": 0.386868686868687, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GLM-5.1 (Non-reasoning)", "shortName": "GLM-5.1 (non-reasoning)", "slug": "glm-5-1-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 24.2412101500726, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.356060606060606, "scicode": null, "hle": 0.279425393883225, "critpt": 0, "lcr": 0.533333333333333, "gpqa": 0.839393939393939, "omniscience": -22.433333333333334, "omniscienceAccuracy": 0.25183333333333335, "omniscienceNonHallucination": 0.36355535754065493, "gdpvalNormalized": null, "price1mInputTokens": 1.38, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.26, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 52.0976927311639, "medianTimeToFirstTokenSeconds": 1.75800576099999, "isOpenWeights": true }, { "name": "Gemini 1.5 Flash-8B", "shortName": "Gemini 1.5 Flash-8B", "slug": "gemini-1-5-flash-8b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 6.15335094096311, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0469, "critpt": null, "lcr": null, "gpqa": 0.358585858585859, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1048576, "isOpenWeights": false }, { "name": "GLM-4.6 (Non-reasoning)", "shortName": "GLM-4.6 (non-reasoning)", "slug": "glm-4-6", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 14.9292992892622, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.287878787878788, "scicode": null, "hle": 0.0546802594995366, "critpt": 0, "lcr": 0.263333333333333, "gpqa": 0.632323232323232, "omniscience": -31.683333333333334, "omniscienceAccuracy": 0.21416666666666667, "omniscienceNonHallucination": 0.32428419936373276, "gdpvalNormalized": null, "price1mInputTokens": 0.575, "price1mOutputTokens": 2.2, "cacheHitPrice": 0.355, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 32.9747344016304, "medianTimeToFirstTokenSeconds": 5.01067792850005, "isOpenWeights": true }, { "name": "Grok 4.3 (High)", "shortName": "Grok 4.3 (high)", "slug": "grok-4-3", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 24.8801191398002, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.20537871205068303, "terminalBench40": 0, "terminalBench21": 0.397003745318352, "terminalbenchHard": 0.378787878787879, "scicode": 0.482638888888889, "hle": 0.372103799814643, "critpt": 0.08, "lcr": 0.73, "gpqa": 0.901010101010101, "omniscience": 17.983333333333334, "omniscienceAccuracy": 0.3478333333333333, "omniscienceNonHallucination": 0.7423971377459749, "gdpvalNormalized": 0.21160500000000002, "price1mInputTokens": 1.25, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.2, "cacheWritePrice": 1.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 113.337969181879, "medianTimeToFirstTokenSeconds": 26.733500713, "isOpenWeights": false }, { "name": "Olmo 3.1 32B Instruct", "shortName": "Olmo 3.1 32B Instruct", "slug": "olmo-3-1-32b-instruct", "isReasoning": false, "deprecated": false, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 6.49671923993574, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0500463392029657, "critpt": 0, "lcr": 0, "gpqa": 0.539393939393939, "omniscience": -52.36666666666667, "omniscienceAccuracy": 0.1205, "omniscienceNonHallucination": 0.2675762743983324, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "MiMo-V2-Omni", "shortName": "MiMo-V2-Omni", "slug": "mimo-v2-omni", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 23.9174211586356, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.348484848484849, "scicode": null, "hle": 0.221037998146432, "critpt": 0.0114285714285714, "lcr": 0.75, "gpqa": 0.828282828282828, "omniscience": -20.133333333333333, "omniscienceAccuracy": 0.193, "omniscienceNonHallucination": 0.5113589425857084, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Ministral 3 8B", "shortName": "Ministral 3 8B", "slug": "ministral-3-8b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 5.47767370454416, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.011086416592507111, "terminalBench40": 0, "terminalBench21": 0.0411985018726592, "terminalbenchHard": 0.0454545454545455, "scicode": 0.207175925925926, "hle": 0.0426320667284523, "critpt": 0, "lcr": 0.256666666666667, "gpqa": 0.470707070707071, "omniscience": -68.95, "omniscienceAccuracy": 0.12983333333333333, "omniscienceNonHallucination": 0.05841792760007658, "gdpvalNormalized": 0, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.15, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 95.5607305126204, "medianTimeToFirstTokenSeconds": 0.777167565000027, "isOpenWeights": true }, { "name": "Claude Sonnet 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)", "shortName": "Claude Sonnet 5.5 (xhigh with fallback)", "slug": "claude-sonnet-5-5-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 51.8522867021152, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.742576603824205, "terminalBench40": 0.570707070707071, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.572916666666667, "hle": 0.5, "critpt": 0.311428571428571, "lcr": 0.796666666666667, "gpqa": null, "omniscience": 23.5, "omniscienceAccuracy": 0.5303333333333333, "omniscienceNonHallucination": 0.37118523775727463, "gdpvalNormalized": 0.61245, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 103.111043384427, "medianTimeToFirstTokenSeconds": 31.9748528964999, "isOpenWeights": false }, { "name": "Llama 3.3 Instruct 70B", "shortName": "Llama 3.3 70B", "slug": "llama-3-3-instruct-70b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 7.66380859299009, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.048689138576779, "terminalbenchHard": 0.0303030303030303, "scicode": null, "hle": 0.0356, "critpt": 0, "lcr": 0.156666666666667, "gpqa": 0.497979797979798, "omniscience": -54.166666666666664, "omniscienceAccuracy": 0.1895, "omniscienceNonHallucination": 0.09788196586469256, "gdpvalNormalized": 0, "price1mInputTokens": 0.71, "price1mOutputTokens": 0.72, "cacheHitPrice": 0.71, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 81.9508627841557, "medianTimeToFirstTokenSeconds": 1.63151134599991, "isOpenWeights": true }, { "name": "Nemotron Cascade 2 30B A3B", "shortName": "Nemotron Cascade 2 30B A3B", "slug": "nemotron-cascade-2-30b-a3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 11.6708359406601, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.205992509363296, "terminalbenchHard": 0.212121212121212, "scicode": null, "hle": 0.120018535681186, "critpt": 0.00571428571428571, "lcr": 0.393333333333333, "gpqa": 0.757575757575758, "omniscience": -51.46666666666667, "omniscienceAccuracy": 0.18283333333333332, "omniscienceNonHallucination": 0.14644095451764227, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "R1 1776", "shortName": "R1 1776", "slug": "r1-1776", "isReasoning": true, "deprecated": false, "modelCreatorName": "Perplexity", "intelligenceIndex": 6.43857605749744, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Claude Opus 4.8 (Adaptive Reasoning, Max Effort)", "shortName": "Claude Opus 4.8 (max)", "slug": "claude-opus-4-8", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 41.7899103794639, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 4.080976976477826, "terminalBench40": 0.217171717171717, "terminalBench21": 0.846441947565543, "terminalbenchHard": 0.583333333333333, "scicode": 0.543981481481482, "hle": 0.486561631139944, "critpt": 0.208571428571429, "lcr": 0.776666666666667, "gpqa": 0.92020202020202, "omniscience": 28.75, "omniscienceAccuracy": 0.48833333333333334, "omniscienceNonHallucination": 0.6074918566775245, "gdpvalNormalized": 0.46915999999999997, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 58.9275767468846, "medianTimeToFirstTokenSeconds": 62.2197937340001, "isOpenWeights": false }, { "name": "Muse Spark 1.3 (Max)", "shortName": "Muse Spark 1.3 (max)", "slug": "muse-spark-1-3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 48.0923107719685, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.6048932100125866, "terminalBench40": 0.333333333333333, "terminalBench21": 0.842696629213483, "terminalbenchHard": null, "scicode": 0.587962962962963, "hle": 0.487025023169602, "critpt": 0.248571428571429, "lcr": 0.83, "gpqa": 0.935353535353535, "omniscience": 25, "omniscienceAccuracy": 0.43583333333333335, "omniscienceNonHallucination": 0.670605612998523, "gdpvalNormalized": 0.5861649999999999, "price1mInputTokens": 1.25, "price1mOutputTokens": 4.25, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 181.327973304986, "medianTimeToFirstTokenSeconds": 37.3304717435, "isOpenWeights": false }, { "name": "Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)", "shortName": "Claude Sonnet 5 (xhigh)", "slug": "claude-sonnet-5-xhigh", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 34.384211557212, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.8720437784323134, "terminalBench40": 0.0707070707070707, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.540509259259259, "hle": 0.39017608897127, "critpt": 0.154285714285714, "lcr": 0.766666666666667, "gpqa": null, "omniscience": 3.183333333333333, "omniscienceAccuracy": 0.38966666666666666, "omniscienceNonHallucination": 0.4137083560895686, "gdpvalNormalized": 0.4222, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 65.0964145177571, "medianTimeToFirstTokenSeconds": 16.4349281564999, "isOpenWeights": false }, { "name": "Qwen3.8-Flash-Next", "shortName": "Qwen3.8-Flash-Next", "slug": "qwen3-8-flash-next", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 39.8223021864741, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.3721760025302965, "terminalBench40": 0.252525252525253, "terminalBench21": 0.861423220973783, "terminalbenchHard": null, "scicode": 0.505787037037037, "hle": 0.380444856348471, "critpt": 0.111428571428571, "lcr": 0.796666666666667, "gpqa": 0.923232323232323, "omniscience": -9.716666666666667, "omniscienceAccuracy": 0.245, "omniscienceNonHallucination": 0.5467991169977925, "gdpvalNormalized": 0.556065, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.47, "cacheHitPrice": 0.016, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 58.0258672918294, "medianTimeToFirstTokenSeconds": 2.49549601049999, "isOpenWeights": true }, { "name": "Claude Fable 5.1 (Adaptive Reasoning, Xhigh Effort, Default Fallback)", "shortName": "Claude Fable 5.1 (xhigh with fallback)", "slug": "claude-fable-5-1-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 53.2032587538944, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 5.978304247388257, "terminalBench40": 0.55050505050505, "terminalBench21": 0.910112359550562, "terminalbenchHard": null, "scicode": 0.608796296296296, "hle": 0.587117701575533, "critpt": 0.311428571428571, "lcr": 0.83, "gpqa": 0.934343434343434, "omniscience": 42.38333333333333, "omniscienceAccuracy": 0.6621666666666667, "omniscienceNonHallucination": 0.2945239269856932, "gdpvalNormalized": 0.61046, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 0.25, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 61.177965386725, "medianTimeToFirstTokenSeconds": 102.128350551, "isOpenWeights": false }, { "name": "GPT-5.6 Terra (Non-reasoning)", "shortName": "GPT-5.6 Terra (non-reasoning)", "slug": "gpt-5-6-terra-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.7797013584503, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.13972397093498629, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.561797752808989, "terminalbenchHard": null, "scicode": 0.451388888888889, "hle": 0.113994439295644, "critpt": 0.02, "lcr": 0.586666666666667, "gpqa": 0.746464646464646, "omniscience": -23.216666666666665, "omniscienceAccuracy": 0.36816666666666664, "omniscienceNonHallucination": 0.04985491954629384, "gdpvalNormalized": 0.292385, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 80.8849592109737, "medianTimeToFirstTokenSeconds": 0.797963476500001, "isOpenWeights": false }, { "name": "MiniMax M1 80k", "shortName": "MiniMax M1 80k", "slug": "minimax-m1-80k", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "intelligenceIndex": 11.725328542655, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0303030303030303, "scicode": null, "hle": 0.0889712696941613, "critpt": 0, "lcr": 0.576666666666667, "gpqa": 0.696969696969697, "omniscience": -48.5, "omniscienceAccuracy": 0.22466666666666665, "omniscienceNonHallucination": 0.08469475494411005, "gdpvalNormalized": null, "price1mInputTokens": 0.55, "price1mOutputTokens": 2.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "Llama 3.2 Instruct 90B (Vision)", "shortName": "Llama 3.2 90B (Vision)", "slug": "llama-3-2-instruct-90b-vision", "isReasoning": false, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 6.4088295766243, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0449, "critpt": null, "lcr": null, "gpqa": 0.432323232323232, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "K-EXAONE (Non-reasoning)", "shortName": "K-EXAONE (non-reasoning)", "slug": "k-exaone-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "LG AI Research", "intelligenceIndex": 11.2077008943064, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0569972196478221, "critpt": 0, "lcr": 0.533333333333333, "gpqa": 0.694949494949495, "omniscience": -65.4, "omniscienceAccuracy": 0.1365, "omniscienceNonHallucination": 0.08453966415749858, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Claude Fable 5.1 (Adaptive Reasoning, High Effort, Default Fallback)", "shortName": "Claude Fable 5.1 (high with fallback)", "slug": "claude-fable-5-1-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 51.1515708852237, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 3.912526554325558, "terminalBench40": 0.52020202020202, "terminalBench21": 0.898876404494382, "terminalbenchHard": null, "scicode": 0.586805555555556, "hle": 0.559314179796108, "critpt": 0.302857142857143, "lcr": 0.836666666666667, "gpqa": 0.906060606060606, "omniscience": 40.8, "omniscienceAccuracy": 0.6493333333333333, "omniscienceNonHallucination": 0.311787072243346, "gdpvalNormalized": 0.558645, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 0.25, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 50.9733201011956, "medianTimeToFirstTokenSeconds": 27.3971238795, "isOpenWeights": false }, { "name": "Qwen3 8B (Reasoning)", "shortName": "Qwen3 8B", "slug": "qwen3-8b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.27634965725808, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.0224719101123595, "terminalbenchHard": 0.0227272727272727, "scicode": 0.278935185185185, "hle": 0.0389, "critpt": 0, "lcr": 0, "gpqa": 0.588888888888889, "omniscience": -64.73333333333333, "omniscienceAccuracy": 0.13633333333333333, "omniscienceNonHallucination": 0.09262832883056737, "gdpvalNormalized": 0, "price1mInputTokens": 0.18, "price1mOutputTokens": 2.1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 41.3398748303785, "medianTimeToFirstTokenSeconds": 3.81179915000001, "isOpenWeights": true }, { "name": "Claude 2.1", "shortName": "Claude 2.1", "slug": "claude-21", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 5.58581054715502, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0389, "critpt": null, "lcr": null, "gpqa": 0.319191919191919, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "GPT-5.6 Terra (Xhigh)", "shortName": "GPT-5.6 Terra (xhigh)", "slug": "gpt-5-6-terra-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 37.9513370742927, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.6317911346924275, "terminalBench40": 0.101010101010101, "terminalBench21": 0.801498127340824, "terminalbenchHard": 0.628787878787879, "scicode": 0.523148148148148, "hle": 0.418906394810009, "critpt": 0.271428571428571, "lcr": 0.79, "gpqa": 0.908080808080808, "omniscience": -2.9833333333333334, "omniscienceAccuracy": 0.45516666666666666, "omniscienceNonHallucination": 0.10981951667176504, "gdpvalNormalized": 0.46408000000000005, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 80.381758504429, "medianTimeToFirstTokenSeconds": 28.920639036, "isOpenWeights": false }, { "name": "K2 Horizon 0.9B", "shortName": "K2 Horizon 0.9B", "slug": "k2-horizon-0-9b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 3.04760621054014, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.0112359550561798, "terminalbenchHard": null, "scicode": 0.0694444444444444, "hle": 0.0537534754402224, "critpt": 0, "lcr": 0.0633333333333333, "gpqa": 0.292929292929293, "omniscience": -72.81666666666666, "omniscienceAccuracy": 0.06933333333333333, "omniscienceNonHallucination": 0.14308739255014324, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Devstral Small (Jul '25)", "shortName": "Devstral Small", "slug": "devstral-small", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 7.60237445255949, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0375347544022243, "critpt": 0, "lcr": 0.186666666666667, "gpqa": 0.414141414141414, "omniscience": -50.833333333333336, "omniscienceAccuracy": 0.14733333333333334, "omniscienceNonHallucination": 0.2310398749022674, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "NVIDIA Nemotron 3 Nano 4B", "shortName": "Nemotron 3 Nano 4B", "slug": "nvidia-nemotron-3-nano-4b", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.38161312356127, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.0374531835205993, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0486561631139944, "critpt": 0, "lcr": 0.216666666666667, "gpqa": 0.513131313131313, "omniscience": -70.53333333333333, "omniscienceAccuracy": 0.09566666666666666, "omniscienceNonHallucination": 0.1142646516771102, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262000, "isOpenWeights": true }, { "name": "GPT-4.1 mini", "shortName": "GPT-4.1 mini", "slug": "gpt-4-1-mini", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 10.1588744766829, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.101123595505618, "terminalbenchHard": 0.0757575757575758, "scicode": null, "hle": 0.0502, "critpt": 0, "lcr": 0.44, "gpqa": 0.663636363636364, "omniscience": -53.56666666666667, "omniscienceAccuracy": 0.20316666666666666, "omniscienceNonHallucination": 0.07278811964024268, "gdpvalNormalized": 0, "price1mInputTokens": 0.4, "price1mOutputTokens": 1.6, "cacheHitPrice": 0.1, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 147.220718811219, "medianTimeToFirstTokenSeconds": 0.764643381499979, "isOpenWeights": false }, { "name": "Phi-4 Mini Instruct", "shortName": "Phi-4 Mini", "slug": "phi-4-mini", "isReasoning": false, "deprecated": false, "modelCreatorName": "Microsoft", "intelligenceIndex": 6.32860508899613, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.00374531835205993, "terminalbenchHard": 0, "scicode": null, "hle": 0.0445, "critpt": 0, "lcr": 0.153333333333333, "gpqa": 0.331313131313131, "omniscience": -61.083333333333336, "omniscienceAccuracy": 0.095, "omniscienceNonHallucination": 0.2200736648250461, "gdpvalNormalized": 0, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 45.0666056597243, "medianTimeToFirstTokenSeconds": 0.820092612499963, "isOpenWeights": true }, { "name": "Grok Beta", "shortName": "Grok Beta", "slug": "grok-beta", "isReasoning": false, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 6.88772203061345, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0445, "critpt": null, "lcr": null, "gpqa": 0.470707070707071, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Grok Code Fast 1", "shortName": "Grok Code Fast 1", "slug": "grok-code-fast-1", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 14.0584706489017, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.174242424242424, "scicode": null, "hle": 0.0801668211306766, "critpt": 0, "lcr": 0.53, "gpqa": 0.727272727272727, "omniscience": -37.06666666666667, "omniscienceAccuracy": 0.2355, "omniscienceNonHallucination": 0.20710704163941573, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Nova 2.0 Pro Preview (Non-reasoning)", "shortName": "Nova 2.0 Pro Preview (non-reasoning)", "slug": "nova-2-0-pro", "isReasoning": false, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 9.95904965562327, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.172284644194757, "terminalbenchHard": 0.166666666666667, "scicode": null, "hle": 0.0393883225208526, "critpt": 0, "lcr": 0.3, "gpqa": 0.636363636363636, "omniscience": -49.5, "omniscienceAccuracy": 0.16916666666666666, "omniscienceNonHallucination": 0.20060180541624872, "gdpvalNormalized": 0, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 105.522244677086, "medianTimeToFirstTokenSeconds": 1.06917305299998, "isOpenWeights": false }, { "name": "Ring-1T", "shortName": "Ring-1T", "slug": "ring-1t", "isReasoning": true, "deprecated": true, "modelCreatorName": "InclusionAI", "intelligenceIndex": 10.8974845417933, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.111214087117702, "critpt": 0.00571428571428571, "lcr": 0.546666666666667, "gpqa": 0.773737373737374, "omniscience": -50.916666666666664, "omniscienceAccuracy": 0.20466666666666666, "omniscienceNonHallucination": 0.10247275775356246, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Granite 4.0 1B", "shortName": "Granite 4.0 1B", "slug": "granite-4-0-nano-1b", "isReasoning": false, "deprecated": true, "modelCreatorName": "IBM", "intelligenceIndex": 5.01437141074802, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0481927710843374, "critpt": 0, "lcr": 0.06, "gpqa": 0.280808080808081, "omniscience": -81.61666666666666, "omniscienceAccuracy": 0.0615, "omniscienceNonHallucination": 0.0648197478245427, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-6.1 Sol (Medium)", "shortName": "GPT-6.1 Sol (medium)", "slug": "gpt-6-1-sol-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 47.7833271274065, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.21370088267875537, "terminalBench40": 0.47979797979798, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.532407407407407, "hle": 0.498609823911029, "critpt": 0.277142857142857, "lcr": 0.833333333333333, "gpqa": null, "omniscience": 39.95, "omniscienceAccuracy": 0.604, "omniscienceNonHallucination": 0.48358585858585856, "gdpvalNormalized": 0.46647000000000005, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.1, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 59.4177421720828, "medianTimeToFirstTokenSeconds": 5.291019151, "isOpenWeights": false }, { "name": "Motif 3 (Beta)", "shortName": "Motif 3 (Beta)", "slug": "motif-0714", "isReasoning": true, "deprecated": true, "modelCreatorName": "Motif Technologies", "intelligenceIndex": 32.3147020976362, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.707865168539326, "terminalbenchHard": null, "scicode": null, "hle": 0.403614457831325, "critpt": 0.06, "lcr": 0.746666666666667, "gpqa": 0.868686868686869, "omniscience": -17.766666666666666, "omniscienceAccuracy": 0.21016666666666667, "omniscienceNonHallucination": 0.5089681367377084, "gdpvalNormalized": 0.29323, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": false }, { "name": "K2-V2 (High)", "shortName": "K2-V2 (high)", "slug": "k2-v2", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 9.87390600007694, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0984848484848485, "scicode": null, "hle": 0.105189990732159, "critpt": 0, "lcr": 0.35, "gpqa": 0.680808080808081, "omniscience": -55.583333333333336, "omniscienceAccuracy": 0.18533333333333332, "omniscienceNonHallucination": 0.09022094926350244, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 512000, "isOpenWeights": true }, { "name": "GPT-6 Astra (Low)", "shortName": "GPT-6 Astra (low)", "slug": "gpt-6-astra-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 45.7819243120341, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.8175139285656057, "terminalBench40": 0.419191919191919, "terminalBench21": 0.880149812734082, "terminalbenchHard": null, "scicode": 0.540509259259259, "hle": 0.492122335495829, "critpt": 0.262857142857143, "lcr": 0.8, "gpqa": 0.931313131313131, "omniscience": 40.55, "omniscienceAccuracy": 0.5953333333333334, "omniscienceNonHallucination": 0.5308896210873146, "gdpvalNormalized": 0.43279999999999996, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 1, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 43.1897491817609, "medianTimeToFirstTokenSeconds": 2.96784180250003, "isOpenWeights": false }, { "name": "ERNIE 5.0 Thinking Preview", "shortName": "ERNIE 5.0 Thinking Preview", "slug": "ernie-5-0-thinking-preview", "isReasoning": true, "deprecated": false, "modelCreatorName": "Baidu", "intelligenceIndex": 14.2551784975475, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.25, "scicode": null, "hle": 0.132530120481928, "critpt": 0.0142857142857143, "lcr": null, "gpqa": 0.776767676767677, "omniscience": -45.61666666666667, "omniscienceAccuracy": 0.22133333333333333, "omniscienceNonHallucination": 0.12992294520547942, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Kimi K2.5 (Reasoning)", "shortName": "Kimi K2.5", "slug": "kimi-k2-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 23.458504975068, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.456928838951311, "terminalbenchHard": 0.348484848484849, "scicode": null, "hle": 0.307228915662651, "critpt": 0.0314285714285714, "lcr": 0.78, "gpqa": 0.878787878787879, "omniscience": -7.333333333333333, "omniscienceAccuracy": 0.3521666666666667, "omniscienceNonHallucination": 0.3431952662721893, "gdpvalNormalized": 0.16202999999999998, "price1mInputTokens": 0.6, "price1mOutputTokens": 2.75, "cacheHitPrice": 0.35, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Claude 4.5 Haiku (Reasoning)", "shortName": "Claude 4.5 Haiku", "slug": "claude-4-5-haiku-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 16.8822352291856, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.27682075830176456, "terminalBench40": 0, "terminalBench21": 0.441947565543071, "terminalbenchHard": 0.272727272727273, "scicode": 0.422453703703704, "hle": 0.103799814643188, "critpt": 0, "lcr": 0.743333333333333, "gpqa": 0.671717171717172, "omniscience": -4.366666666666666, "omniscienceAccuracy": 0.18016666666666667, "omniscienceNonHallucination": 0.7269770278511893, "gdpvalNormalized": 0.10943000000000001, "price1mInputTokens": 1, "price1mOutputTokens": 5, "cacheHitPrice": 0.1, "cacheWritePrice": 1.25, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 91.099449074885, "medianTimeToFirstTokenSeconds": 17.783194269, "isOpenWeights": false }, { "name": "Granite 4.0 H 350M", "shortName": "Granite 4.0 H 350M", "slug": "granite-4-0-h-350m", "isReasoning": false, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0644114921223355, "critpt": 0, "lcr": 0, "gpqa": 0.256565656565657, "omniscience": -80.91666666666667, "omniscienceAccuracy": 0.038, "omniscienceNonHallucination": 0.11936936936936937, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Qwen3.5 0.8B (Reasoning)", "shortName": "Qwen3.5 0.8B", "slug": "qwen3-5-0-8b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.12847840160203, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0, "terminalbenchHard": 0, "scicode": null, "hle": 0.0106580166821131, "critpt": 0, "lcr": 0.09, "gpqa": 0.111111111111111, "omniscience": -54.516666666666666, "omniscienceAccuracy": 0.042333333333333334, "omniscienceNonHallucination": 0.38652975983292726, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Apriel-v1.6-15B-Thinker", "shortName": "Apriel-v1.6-15B-Thinker", "slug": "apriel-v1-6-15b-thinker", "isReasoning": true, "deprecated": false, "modelCreatorName": "ServiceNow", "intelligenceIndex": 13.401396407375, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.143939393939394, "scicode": null, "hle": 0.107970342910102, "critpt": 0.00285714285714286, "lcr": null, "gpqa": 0.733333333333333, "omniscience": -59.36666666666667, "omniscienceAccuracy": 0.16966666666666666, "omniscienceNonHallucination": 0.0806904857486953, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Claude 4 Sonnet (Reasoning)", "shortName": "Claude 4 Sonnet", "slug": "claude-4-sonnet-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 18.9183251073899, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.363295880149813, "terminalbenchHard": 0.310606060606061, "scicode": null, "hle": 0.107043558850788, "critpt": 0.00285714285714286, "lcr": 0.703333333333333, "gpqa": 0.776767676767677, "omniscience": 0.16666666666666666, "omniscienceAccuracy": 0.22683333333333333, "omniscienceNonHallucination": 0.7087734425522743, "gdpvalNormalized": 0.08988, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Gemma 3n E4B Instruct", "shortName": "Gemma 3n E4B", "slug": "gemma-3n-e4b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.00749063670411985, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0449490268767377, "critpt": 0, "lcr": 0, "gpqa": 0.295959595959596, "omniscience": -81.33333333333333, "omniscienceAccuracy": 0.07983333333333334, "omniscienceNonHallucination": 0.029342510414779976, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Qwen3 VL 235B A22B (Reasoning)", "shortName": "Qwen3 VL 235B A22B", "slug": "qwen3-vl-235b-a22b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 13.4375683732418, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.113636363636364, "scicode": null, "hle": 0.119091751621872, "critpt": 0, "lcr": 0.636666666666667, "gpqa": 0.771717171717172, "omniscience": -46.55, "omniscienceAccuracy": 0.2085, "omniscienceNonHallucination": 0.1484523057485786, "gdpvalNormalized": null, "price1mInputTokens": 0.4, "price1mOutputTokens": 4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 54.8890755934553, "medianTimeToFirstTokenSeconds": 3.00193515699999, "isOpenWeights": true }, { "name": "DeepSeek R1 0528 (May '25)", "shortName": "DeepSeek R1 0528", "slug": "deepseek-r1", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 13.1220574118753, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.159090909090909, "scicode": null, "hle": 0.158480074142725, "critpt": 0.0142857142857143, "lcr": 0.556666666666667, "gpqa": 0.813131313131313, "omniscience": -27.433333333333334, "omniscienceAccuracy": 0.30533333333333335, "omniscienceNonHallucination": 0.16554702495201534, "gdpvalNormalized": null, "price1mInputTokens": 1.35, "price1mOutputTokens": 3, "cacheHitPrice": 1.35, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3.7 Plus", "shortName": "Qwen3.7 Plus", "slug": "qwen3-7-plus", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 25.1622215821984, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.21702592259078568, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.610486891385768, "terminalbenchHard": 0.46969696969697, "scicode": 0.460648148148148, "hle": 0.356348470806302, "critpt": 0.0914285714285714, "lcr": 0.73, "gpqa": 0.9, "omniscience": 1.0833333333333333, "omniscienceAccuracy": 0.22516666666666665, "omniscienceNonHallucination": 0.7233813723381373, "gdpvalNormalized": 0.12802, "price1mInputTokens": 0.4, "price1mOutputTokens": 1.6, "cacheHitPrice": 0.08, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 56.3170463722374, "medianTimeToFirstTokenSeconds": 2.17197500649999, "isOpenWeights": false }, { "name": "GPT-5.4 (Xhigh)", "shortName": "GPT-5.4 (xhigh)", "slug": "gpt-5-4", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 38.9755886799637, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.782771535580524, "terminalbenchHard": 0.575757575757576, "scicode": null, "hle": 0.437442075996293, "critpt": 0.2342857143, "lcr": 0.82, "gpqa": 0.92020202020202, "omniscience": 5.783333333333333, "omniscienceAccuracy": 0.5085, "omniscienceNonHallucination": 0.08307900983384198, "gdpvalNormalized": 0.366335, "price1mInputTokens": 2.5, "price1mOutputTokens": 15, "cacheHitPrice": 0.25, "cacheWritePrice": null, "contextWindowTokens": 1050000, "medianOutputTokensPerSecond": 95.6021298466076, "medianTimeToFirstTokenSeconds": 188.4334878525, "isOpenWeights": false }, { "name": "Gemma 3n E2B Instruct", "shortName": "Gemma 3n E2B", "slug": "gemma-3n-e2b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0417052826691381, "critpt": 0, "lcr": 0, "gpqa": 0.229292929292929, "omniscience": -80.16666666666667, "omniscienceAccuracy": 0.06833333333333333, "omniscienceNonHallucination": 0.0661896243291592, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "DeepHermes 3 - Mistral 24B Preview (Non-reasoning)", "shortName": "DeepHermes 3 - Mistral 24B (non-reasoning)", "slug": "deephermes-3-mistral-24b-preview", "isReasoning": false, "deprecated": false, "modelCreatorName": "Nous Research", "intelligenceIndex": 6.07426212371414, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0376, "critpt": null, "lcr": null, "gpqa": 0.381818181818182, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Reka Flash (Sep '24)", "shortName": "Reka Flash", "slug": "reka-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "Reka AI", "intelligenceIndex": 6.43110884456904, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "GPT-5.6 Luna (High)", "shortName": "GPT-5.6 Luna (high)", "slug": "gpt-5-6-luna-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 32.1196453501091, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.043981552251920096, "terminalBench40": 0.0252525252525253, "terminalBench21": 0.696629213483146, "terminalbenchHard": null, "scicode": 0.516203703703704, "hle": 0.334105653382762, "critpt": 0.165714285714286, "lcr": 0.803333333333333, "gpqa": 0.891919191919192, "omniscience": -12, "omniscienceAccuracy": 0.41783333333333333, "omniscienceNonHallucination": 0.0761523046092184, "gdpvalNormalized": 0.40938, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.02, "cacheWritePrice": 0.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 99.7231372526601, "medianTimeToFirstTokenSeconds": 14.759375466, "isOpenWeights": false }, { "name": "Llama 3 Instruct 70B", "shortName": "Llama 3 70B", "slug": "llama-3-instruct-70b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 5.45277882640287, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0452, "critpt": 0, "lcr": 0, "gpqa": 0.378787878787879, "omniscience": -54.31666666666667, "omniscienceAccuracy": 0.17716666666666667, "omniscienceNonHallucination": 0.12456957666599144, "gdpvalNormalized": null, "price1mInputTokens": 0.65, "price1mOutputTokens": 2.75, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "NVIDIA Nemotron Nano 12B v2 VL (Non-reasoning)", "shortName": "NVIDIA Nemotron Nano 12B v2 VL (non-reasoning)", "slug": "nvidia-nemotron-nano-12b-v2-vl", "isReasoning": false, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 5.82045753777258, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0426320667284523, "critpt": 0, "lcr": 0.206666666666667, "gpqa": 0.439393939393939, "omniscience": -72.23333333333333, "omniscienceAccuracy": 0.11916666666666667, "omniscienceNonHallucination": 0.04465468306527909, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 202.094510648999, "medianTimeToFirstTokenSeconds": 1.11880145099998, "isOpenWeights": true }, { "name": "Devstral Medium", "shortName": "Devstral Medium", "slug": "devstral-medium", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 9.0057933497923, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0909090909090909, "scicode": null, "hle": 0.0375347544022243, "critpt": 0, "lcr": 0.32, "gpqa": 0.491919191919192, "omniscience": -31.566666666666666, "omniscienceAccuracy": 0.18883333333333333, "omniscienceNonHallucination": 0.37805629751386893, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Command-R (Mar '24)", "shortName": "Command-R (Mar)", "slug": "command-r-03-2024", "isReasoning": false, "deprecated": true, "modelCreatorName": "Cohere", "intelligenceIndex": 5.03498213171439, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0479, "critpt": null, "lcr": null, "gpqa": 0.283838383838384, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Apriel-v1.5-15B-Thinker", "shortName": "Apriel-v1.5-15B-Thinker", "slug": "apriel-v1-5-15b-thinker", "isReasoning": true, "deprecated": true, "modelCreatorName": "ServiceNow", "intelligenceIndex": 13.824080582574, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.106060606060606, "scicode": null, "hle": 0.121408711770158, "critpt": 0.0114285714285714, "lcr": 0.213333333333333, "gpqa": 0.713131313131313, "omniscience": -54.8, "omniscienceAccuracy": 0.1645, "omniscienceNonHallucination": 0.14721723518850982, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 1.5 Flash (May '24)", "shortName": "Gemini 1.5 Flash (May)", "slug": "gemini-1-5-flash-may-2024", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 5.93750542104558, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0436, "critpt": null, "lcr": null, "gpqa": 0.324242424242424, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Claude 3.5 Sonnet (June '24)", "shortName": "Claude 3.5 Sonnet (June)", "slug": "claude-35-sonnet-june-24", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 7.20987771468693, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0323, "critpt": null, "lcr": null, "gpqa": 0.55959595959596, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Step3 VL 10B", "shortName": "Step3 VL 10B", "slug": "step-3-vl-10b", "isReasoning": true, "deprecated": false, "modelCreatorName": "StepFun", "intelligenceIndex": 7.69391371030502, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.053030303030303, "scicode": null, "hle": 0.108433734939759, "critpt": 0, "lcr": 0, "gpqa": 0.68989898989899, "omniscience": -59, "omniscienceAccuracy": 0.13266666666666665, "omniscienceNonHallucination": 0.16679477325134517, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "DeepSeek V4 Flash 0420 (Non-reasoning)", "shortName": "DeepSeek V4 Flash (non-reasoning)", "slug": "deepseek-v4-flash-0420-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 18.8769056307071, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.340909090909091, "scicode": null, "hle": 0.0778498609823911, "critpt": 0.00285714285714286, "lcr": 0.416666666666667, "gpqa": 0.716161616161616, "omniscience": -43.96666666666667, "omniscienceAccuracy": 0.26, "omniscienceNonHallucination": 0.0545045045045045, "gdpvalNormalized": null, "price1mInputTokens": 0.0919, "price1mOutputTokens": 0.1913, "cacheHitPrice": 0.0545, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "Llama 4 Scout", "shortName": "Llama 4 Scout", "slug": "llama-4-scout", "isReasoning": false, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 8.07503439053102, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.0374531835205993, "terminalbenchHard": 0.0151515151515152, "scicode": 0.212962962962963, "hle": 0.0378, "critpt": 0, "lcr": 0.277, "gpqa": 0.586868686868687, "omniscience": -52.15, "omniscienceAccuracy": 0.15166666666666667, "omniscienceNonHallucination": 0.20648330058939102, "gdpvalNormalized": 0, "price1mInputTokens": 0.19, "price1mOutputTokens": 0.6799999999999999, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 10000000, "medianOutputTokensPerSecond": 128.778987582265, "medianTimeToFirstTokenSeconds": 0.844021005499982, "isOpenWeights": true }, { "name": "Granite 4.2 8B", "shortName": "Granite 4.2 8B", "slug": "granite-4-2-8b", "isReasoning": true, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 11.0812712267815, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.023596091491929188, "terminalBench40": 0, "terminalBench21": 0.183520599250936, "terminalbenchHard": null, "scicode": 0.314814814814815, "hle": 0.0968489341983318, "critpt": 0.00285714285714286, "lcr": 0.45, "gpqa": 0.631313131313131, "omniscience": -17.166666666666668, "omniscienceAccuracy": 0.11216666666666666, "omniscienceNonHallucination": 0.6803078655903886, "gdpvalNormalized": 0, "price1mInputTokens": 0.06, "price1mOutputTokens": 0.25, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 65.3151746981461, "medianTimeToFirstTokenSeconds": 0.730499055499977, "isOpenWeights": true }, { "name": "GPT-5.4 mini (Non-reasoning)", "shortName": "GPT-5.4 mini (non-reasoning)", "slug": "gpt-5-4-mini-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 11.1447193424847, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.181818181818182, "scicode": null, "hle": 0.0588507877664504, "critpt": 0, "lcr": 0.37, "gpqa": 0.606060606060606, "omniscience": -45.416666666666664, "omniscienceAccuracy": 0.2565, "omniscienceNonHallucination": 0.04416050212956735, "gdpvalNormalized": 0.03845999999999998, "price1mInputTokens": 0.75, "price1mOutputTokens": 4.5, "cacheHitPrice": 0.075, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 181.297411773062, "medianTimeToFirstTokenSeconds": 0.855862991500004, "isOpenWeights": false }, { "name": "ERNIE 4.5 300B A47B", "shortName": "ERNIE 4.5 300B A47B", "slug": "ernie-4-5-300b-a47b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Baidu", "intelligenceIndex": 7.50448042633837, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0333642261353105, "critpt": 0, "lcr": null, "gpqa": 0.811111111111111, "omniscience": -35.13333333333333, "omniscienceAccuracy": 0.1905, "omniscienceNonHallucination": 0.33065678402305954, "gdpvalNormalized": null, "price1mInputTokens": 0.28, "price1mOutputTokens": 1.1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "DeepSeek V3.2 Exp (Reasoning)", "shortName": "DeepSeek V3.2 Exp", "slug": "deepseek-v3-2-reasoning-0925", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 16.579440992188, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.310606060606061, "scicode": null, "hle": 0.148748841519926, "critpt": 0.0142857142857143, "lcr": 0.723333333333333, "gpqa": 0.796969696969697, "omniscience": -30.966666666666665, "omniscienceAccuracy": 0.2755, "omniscienceNonHallucination": 0.1923165401426271, "gdpvalNormalized": null, "price1mInputTokens": 0.28, "price1mOutputTokens": 0.42, "cacheHitPrice": 0.028, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Molmo 7B-D", "shortName": "Molmo 7B-D", "slug": "molmo-7b-d", "isReasoning": false, "deprecated": false, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 5.56262049746202, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.051436515291937, "critpt": null, "lcr": 0, "gpqa": 0.24040404040404, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "Qwen3 32B (Reasoning)", "shortName": "Qwen3 32B", "slug": "qwen3-32b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.58694071653199, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.052434456928839, "terminalbenchHard": 0.0303030303030303, "scicode": 0.359953703703704, "hle": 0.0741, "critpt": 0.00285714285714286, "lcr": 0, "gpqa": 0.667676767676768, "omniscience": -50.36666666666667, "omniscienceAccuracy": 0.17433333333333334, "omniscienceNonHallucination": 0.1788453774727493, "gdpvalNormalized": 0, "price1mInputTokens": 0.16, "price1mOutputTokens": 0.64, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 105.552020439417, "medianTimeToFirstTokenSeconds": 2.49645648799998, "isOpenWeights": true }, { "name": "Qwen3.8 27B (Xhigh)", "shortName": "Qwen3.8 27B (xhigh)", "slug": "qwen3-8-27b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 33.6962858057476, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.0073149536565944, "terminalBench40": 0.0555555555555556, "terminalBench21": 0.797752808988764, "terminalbenchHard": null, "scicode": 0.466435185185185, "hle": 0.33920296570899, "critpt": 0.0542857142857143, "lcr": 0.82, "gpqa": 0.905050505050505, "omniscience": -9.983333333333333, "omniscienceAccuracy": 0.15583333333333332, "omniscienceNonHallucination": 0.6971372161895361, "gdpvalNormalized": 0.45542499999999997, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.1, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 44.47717056937, "medianTimeToFirstTokenSeconds": 3.92221442899998, "isOpenWeights": true }, { "name": "Motif 3", "shortName": "Motif 3", "slug": "motif-3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Motif Technologies", "intelligenceIndex": 33.566562209723, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.749063670411985, "terminalbenchHard": null, "scicode": 0.422453703703704, "hle": 0.369786839666358, "critpt": 0.0657142857142857, "lcr": 0.76, "gpqa": 0.834343434343434, "omniscience": 10.183333333333334, "omniscienceAccuracy": 0.3006666666666667, "omniscienceNonHallucination": 0.7156816015252622, "gdpvalNormalized": 0.30933000000000005, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Qwen3 4B 2507 Instruct", "shortName": "Qwen3 4B 2507 (non-reasoning)", "slug": "qwen3-4b-2507-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.74457737576618, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0449490268767377, "critpt": 0, "lcr": 0.113333333333333, "gpqa": 0.517171717171717, "omniscience": -61.93333333333333, "omniscienceAccuracy": 0.109, "omniscienceNonHallucination": 0.18256640478862696, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": true }, { "name": "Gemma 4 E2B (Reasoning)", "shortName": "Gemma 4 E2B", "slug": "gemma-4-e2b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 7.77445525187484, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.00374531835205993, "terminalbenchHard": 0.0303030303030303, "scicode": null, "hle": 0.0477293790546803, "critpt": 0, "lcr": 0.163333333333333, "gpqa": 0.433333333333333, "omniscience": -23.583333333333332, "omniscienceAccuracy": 0.06633333333333333, "omniscienceNonHallucination": 0.6763655837200999, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3.8 Max (0902)", "shortName": "Qwen3.8 Max (0902)", "slug": "qwen3-8-max", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 45.4152084980521, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 5.408509428374016, "terminalBench40": 0.388888888888889, "terminalBench21": 0.887640449438202, "terminalbenchHard": null, "scicode": 0.520833333333333, "hle": 0.430954587581094, "critpt": 0.177142857142857, "lcr": 0.803333333333333, "gpqa": 0.928282828282828, "omniscience": 11.983333333333333, "omniscienceAccuracy": 0.31683333333333336, "omniscienceNonHallucination": 0.7116369846303976, "gdpvalNormalized": 0.581625, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.25, "cacheWritePrice": null, "contextWindowTokens": 983616, "medianOutputTokensPerSecond": 39.3449275530768, "medianTimeToFirstTokenSeconds": 2.98370471699999, "isOpenWeights": false }, { "name": "GPT-5.5 (High)", "shortName": "GPT-5.5 (high)", "slug": "gpt-5-5-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 36.9793737887731, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.541289049705234, "terminalBench40": 0.0909090909090909, "terminalBench21": 0.794007490636704, "terminalbenchHard": 0.598484848484849, "scicode": 0.561342592592593, "hle": 0.450417052826691, "critpt": 0.254285714285714, "lcr": 0.843333333333333, "gpqa": 0.932323232323232, "omniscience": 18.75, "omniscienceAccuracy": 0.5703333333333334, "omniscienceNonHallucination": 0.1089992242048099, "gdpvalNormalized": 0.40851499999999996, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 922000, "medianOutputTokensPerSecond": 85.9715845309457, "medianTimeToFirstTokenSeconds": 15.319479104, "isOpenWeights": false }, { "name": "Claude Sonnet 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)", "shortName": "Claude Sonnet 5.5 (max with fallback)", "slug": "claude-sonnet-5-5", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 55.9779549012591, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 7.62064302108643, "terminalBench40": 0.636363636363636, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.609953703703704, "hle": 0.549582947173309, "critpt": 0.314285714285714, "lcr": 0.826666666666667, "gpqa": null, "omniscience": 32.3, "omniscienceAccuracy": 0.5395, "omniscienceNonHallucination": 0.5298588490770901, "gdpvalNormalized": 0.6721050000000001, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 139.062620193635, "medianTimeToFirstTokenSeconds": 442.3845602705, "isOpenWeights": false }, { "name": "Llama 3.1 Nemotron Nano 4B v1.1 (Reasoning)", "shortName": "Llama 3.1 Nemotron Nano 4B v1.1", "slug": "llama-3-1-nemotron-nano-4b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.3075040848305, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0518999073215941, "critpt": null, "lcr": 0, "gpqa": 0.408080808080808, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 3.7 Flash (High)", "shortName": "Gemini 3.7 Flash (high)", "slug": "gemini-3-7-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 39.0594566404896, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.9253359372385677, "terminalBench40": 0.136363636363636, "terminalBench21": 0.857677902621723, "terminalbenchHard": null, "scicode": 0.571759259259259, "hle": 0.478683966635774, "critpt": 0.142857142857143, "lcr": 0.816666666666667, "gpqa": 0.945454545454545, "omniscience": 26.483333333333334, "omniscienceAccuracy": 0.5531666666666667, "omniscienceNonHallucination": 0.3547183886609474, "gdpvalNormalized": 0.435625, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.075, "cacheWritePrice": 0.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 295.621551933656, "medianTimeToFirstTokenSeconds": 12.0820695095, "isOpenWeights": false }, { "name": "Hy3", "shortName": "Hy3", "slug": "hy3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Tencent", "intelligenceIndex": 25.2972775731571, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.07179763081859954, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.644194756554307, "terminalbenchHard": null, "scicode": 0.486111111111111, "hle": 0.334569045412419, "critpt": 0.0485714285714286, "lcr": 0.79, "gpqa": 0.896969696969697, "omniscience": -18.466666666666665, "omniscienceAccuracy": 0.31966666666666665, "omniscienceNonHallucination": 0.2586967172954434, "gdpvalNormalized": 0.27255999999999997, "price1mInputTokens": 0.136, "price1mOutputTokens": 0.5549999999999999, "cacheHitPrice": 0.034, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 87.6944315706849, "medianTimeToFirstTokenSeconds": 3.08349282349998, "isOpenWeights": true }, { "name": "GPT-6.1 Sol (Max)", "shortName": "GPT-6.1 Sol (max)", "slug": "gpt-6-1-sol", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 51.8332597011541, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.7241670655535033, "terminalBench40": 0.560606060606061, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.541666666666667, "hle": 0.529193697868397, "critpt": 0.317142857142857, "lcr": 0.83, "gpqa": null, "omniscience": 41.53333333333333, "omniscienceAccuracy": 0.621, "omniscienceNonHallucination": 0.4573438874230431, "gdpvalNormalized": 0.5375449999999999, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.1, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 65.2971205036112, "medianTimeToFirstTokenSeconds": 281.943836456, "isOpenWeights": false }, { "name": "DeepSeek R1 Distill Qwen 14B", "shortName": "DeepSeek R1 Distill Qwen 14B", "slug": "deepseek-r1-distill-qwen-14b", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 7.84674495869131, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0413, "critpt": null, "lcr": 0.103333333333333, "gpqa": 0.483838383838384, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3 8B (Non-reasoning)", "shortName": "Qwen3 8B (non-reasoning)", "slug": "qwen3-8b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.99207537851227, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0195, "critpt": 0, "lcr": 0, "gpqa": 0.451515151515152, "omniscience": -73.95, "omniscienceAccuracy": 0.11083333333333334, "omniscienceNonHallucination": 0.0436738519212746, "gdpvalNormalized": null, "price1mInputTokens": 0.18, "price1mOutputTokens": 0.7, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 41.7192585534074, "medianTimeToFirstTokenSeconds": 3.80315321250001, "isOpenWeights": true }, { "name": "Claude 3 Sonnet", "shortName": "Claude 3 Sonnet", "slug": "claude-3-sonnet", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 5.87716084595884, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0362, "critpt": null, "lcr": null, "gpqa": 0.4, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Gemini 3 Pro Preview (Low)", "shortName": "Gemini 3 Pro Preview (low)", "slug": "gemini-3-pro-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 22.3196165324532, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.340909090909091, "scicode": null, "hle": 0.294717330861909, "critpt": 0, "lcr": 0.74, "gpqa": 0.886868686868687, "omniscience": 1.8, "omniscienceAccuracy": 0.4831666666666667, "omniscienceNonHallucination": 0.09996775233795552, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Nova 2.0 Omni (Low)", "shortName": "Nova 2.0 Omni (low)", "slug": "nova-2-0-omni-reasoning-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 11.1140626232787, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": null, "critpt": 0, "lcr": 0.576666666666667, "gpqa": 0.698989898989899, "omniscience": -50.31666666666667, "omniscienceAccuracy": 0.18616666666666667, "omniscienceNonHallucination": 0.15297972557853778, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Arctic Instruct", "shortName": "Arctic", "slug": "arctic-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Snowflake", "intelligenceIndex": 5.4363632246188, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4000, "isOpenWeights": true }, { "name": "GPT-4o (ChatGPT)", "shortName": "GPT-4o (ChatGPT)", "slug": "gpt-4o-chatgpt", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 7.18657581032721, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0273, "critpt": null, "lcr": 0.56, "gpqa": 0.511111111111111, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Jamba 1.5 Large", "shortName": "Jamba 1.5 Large", "slug": "jamba-1-5-large", "isReasoning": false, "deprecated": true, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 6.01209130693622, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0407, "critpt": null, "lcr": null, "gpqa": 0.427272727272727, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Jamba 1.7 Mini", "shortName": "Jamba 1.7 Mini", "slug": "jamba-1-7-mini", "isReasoning": false, "deprecated": false, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 5.21767706986744, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0444856348470806, "critpt": 0, "lcr": 0.16, "gpqa": 0.322222222222222, "omniscience": -73.11666666666666, "omniscienceAccuracy": 0.11266666666666666, "omniscienceNonHallucination": 0.04902329075882794, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 258000, "isOpenWeights": true }, { "name": "Qwen2.5 Instruct 72B", "shortName": "Qwen2.5 72B", "slug": "qwen2-5-72b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.73490670575182, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0356, "critpt": 0, "lcr": null, "gpqa": 0.490909090909091, "omniscience": -53.06666666666667, "omniscienceAccuracy": 0.17466666666666666, "omniscienceNonHallucination": 0.1453957996768982, "gdpvalNormalized": null, "price1mInputTokens": 0.475, "price1mOutputTokens": 0.495, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "o1-mini", "shortName": "o1-mini", "slug": "o1-mini", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 9.77059897024848, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0357, "critpt": null, "lcr": null, "gpqa": 0.603030303030303, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Hy3-preview (Non-reasoning)", "shortName": "Hy3-preview (non-reasoning)", "slug": "hy3-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Tencent", "intelligenceIndex": 17.0361964508377, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.318181818181818, "scicode": null, "hle": 0.0695088044485635, "critpt": 0.00285714285714286, "lcr": 0.42, "gpqa": 0.732323232323232, "omniscience": -34.95, "omniscienceAccuracy": 0.23183333333333334, "omniscienceNonHallucination": 0.24321978737253203, "gdpvalNormalized": null, "price1mInputTokens": 0.063, "price1mOutputTokens": 0.21, "cacheHitPrice": 0.021, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GPT-4o mini", "shortName": "GPT-4o mini", "slug": "gpt-4o-mini", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 6.66432562957107, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0561797752808989, "terminalbenchHard": null, "scicode": null, "hle": 0.042, "critpt": null, "lcr": null, "gpqa": 0.426262626262626, "omniscience": null, "gdpvalNormalized": 0, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.6, "cacheHitPrice": 0.075, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 112.141600800646, "medianTimeToFirstTokenSeconds": 0.820951329000025, "isOpenWeights": false }, { "name": "Trinity Large Thinking", "shortName": "Trinity Large Thinking", "slug": "trinity-large-thinking", "isReasoning": true, "deprecated": false, "modelCreatorName": "Arcee AI", "intelligenceIndex": 10.8242425924218, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.11883788334567012, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.205992509363296, "terminalbenchHard": 0.227272727272727, "scicode": 0.40625, "hle": 0.158480074142725, "critpt": 0.00857142857142857, "lcr": 0.38, "gpqa": 0.751515151515151, "omniscience": -44.06666666666667, "omniscienceAccuracy": 0.225, "omniscienceNonHallucination": 0.14107526881720434, "gdpvalNormalized": 0, "price1mInputTokens": 0.25, "price1mOutputTokens": 0.8, "cacheHitPrice": 0.06, "cacheWritePrice": null, "contextWindowTokens": 512000, "medianOutputTokensPerSecond": 335.67647963896, "medianTimeToFirstTokenSeconds": 1.33106761500001, "isOpenWeights": true }, { "name": "DeepHermes 3 - Llama-3.1 8B Preview (Non-reasoning)", "shortName": "DeepHermes 3 - Llama-3.1 8B (non-reasoning)", "slug": "deephermes-3-llama-3-1-8b-preview", "isReasoning": false, "deprecated": false, "modelCreatorName": "Nous Research", "intelligenceIndex": 5.08032603461277, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0426, "critpt": null, "lcr": null, "gpqa": 0.26969696969697, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Solar Pro 3", "shortName": "Solar Pro 3", "slug": "solar-pro-3", "isReasoning": true, "deprecated": true, "modelCreatorName": "Upstage", "intelligenceIndex": 7.81551744502774, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.07942705217350178, "terminalBench40": 0, "terminalBench21": 0.119850187265918, "terminalbenchHard": 0.0757575757575758, "scicode": 0.25462962962963, "hle": 0.102873030583874, "critpt": 0, "lcr": 0.323333333333333, "gpqa": 0.724242424242424, "omniscience": -53.4, "omniscienceAccuracy": 0.185, "omniscienceNonHallucination": 0.11779141104294477, "gdpvalNormalized": 0, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.6, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 148.400994645868, "medianTimeToFirstTokenSeconds": 1.2907078830001, "isOpenWeights": false }, { "name": "Claude Opus 5 (Adaptive Reasoning, Max Effort)", "shortName": "Claude Opus 5 (max)", "slug": "claude-opus-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 50.7771115797629, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 5.858396237036018, "terminalBench40": 0.48989898989899, "terminalBench21": 0.891385767790262, "terminalbenchHard": null, "scicode": 0.563657407407407, "hle": 0.548656163113994, "critpt": 0.291428571428571, "lcr": 0.793333333333333, "gpqa": 0.932323232323232, "omniscience": 37.06666666666667, "omniscienceAccuracy": 0.6086666666666667, "omniscienceNonHallucination": 0.3918228279386712, "gdpvalNormalized": 0.60396, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 52.4563247590679, "medianTimeToFirstTokenSeconds": 55.6557573785, "isOpenWeights": false }, { "name": "Qwen3 Omni 30B A3B (Reasoning)", "shortName": "Qwen3 Omni 30B A3B", "slug": "qwen3-omni-30b-a3b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.76082961984319, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0746061167747915, "critpt": 0, "lcr": 0, "gpqa": 0.726262626262626, "omniscience": -61.4, "omniscienceAccuracy": 0.146, "omniscienceNonHallucination": 0.11007025761124123, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 0.97, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "medianOutputTokensPerSecond": 102.204662204903, "medianTimeToFirstTokenSeconds": 1.93328877150009, "isOpenWeights": true }, { "name": "Qwen2.5 Max", "shortName": "Qwen2.5 Max", "slug": "qwen-2-5-max", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.01995268843494, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0382, "critpt": null, "lcr": null, "gpqa": 0.586868686868687, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": false }, { "name": "Claude Sonnet 5 (Adaptive Reasoning, High Effort)", "shortName": "Claude Sonnet 5 (high)", "slug": "claude-sonnet-5-high", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 31.6623246664651, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.7924692244111728, "terminalBench40": 0.0505050505050505, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.542824074074074, "hle": 0.357275254865616, "critpt": 0.151428571428571, "lcr": 0.766666666666667, "gpqa": null, "omniscience": -3.683333333333333, "omniscienceAccuracy": 0.37433333333333335, "omniscienceNonHallucination": 0.34283431006925946, "gdpvalNormalized": 0.37286, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 65.024613547758, "medianTimeToFirstTokenSeconds": 13.9945428015, "isOpenWeights": false }, { "name": "Kimi K2.5 (Non-reasoning)", "shortName": "Kimi K2.5 (non-reasoning)", "slug": "kimi-k2-5-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 19.4343313470652, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.189393939393939, "scicode": null, "hle": 0.131603336422614, "critpt": 0.00571428571428571, "lcr": 0.673333333333333, "gpqa": 0.788888888888889, "omniscience": -13.766666666666667, "omniscienceAccuracy": 0.24133333333333334, "omniscienceNonHallucination": 0.5004393673110721, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 3, "cacheHitPrice": 0.6, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GPT-5.4 mini (Xhigh)", "shortName": "GPT-5.4 mini (xhigh)", "slug": "gpt-5-4-mini", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 24.0682169231964, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.4473810333705224, "terminalBench40": 0.0202020202020202, "terminalBench21": 0.591760299625468, "terminalbenchHard": 0.522727272727273, "scicode": 0.520833333333333, "hle": 0.281278962001854, "critpt": 0.1, "lcr": 0.77, "gpqa": 0.874747474747475, "omniscience": -18.916666666666668, "omniscienceAccuracy": 0.37466666666666665, "omniscienceNonHallucination": 0.0983475479744137, "gdpvalNormalized": 0.25011, "price1mInputTokens": 0.75, "price1mOutputTokens": 4.5, "cacheHitPrice": 0.075, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 213.011391682428, "medianTimeToFirstTokenSeconds": 127.720761353, "isOpenWeights": false }, { "name": "K2 Horizon 375B A23B", "shortName": "K2 Horizon 375B A23B", "slug": "k2-horizon-375b-a23b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 30.5015302696723, "intelligenceIndexIsEstimated": false, "terminalBench40": 0.0151515151515152, "terminalBench21": 0.719101123595506, "terminalbenchHard": null, "scicode": 0.429398148148148, "hle": 0.319740500463392, "critpt": 0.0457142857142857, "lcr": 0.8, "gpqa": 0.872727272727273, "omniscience": -2.966666666666667, "omniscienceAccuracy": 0.18183333333333335, "omniscienceNonHallucination": 0.7414952128743124, "gdpvalNormalized": 0.42460000000000003, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 524288, "isOpenWeights": true }, { "name": "Claude Sonnet 5.5 (Adaptive Reasoning, Low Effort, Default Fallback)", "shortName": "Claude Sonnet 5.5 (low with fallback)", "slug": "claude-sonnet-5-5-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": null, "intelligenceIndexIsEstimated": false, "terminalBench40": 0.207070707070707, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.490740740740741, "hle": 0.362372567191844, "critpt": 0.114285714285714, "lcr": 0.76, "gpqa": null, "omniscience": 19.366666666666667, "omniscienceAccuracy": 0.463, "omniscienceNonHallucination": 0.49844816883923027, "gdpvalNormalized": 0.33398, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 88.2074163689369, "medianTimeToFirstTokenSeconds": 1.16662582949999, "isOpenWeights": false }, { "name": "Gemini 3.1 Pro Preview", "shortName": "Gemini 3.1 Pro Preview", "slug": "gemini-3-1-pro-preview", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 29.7185656132261, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.6747317275695374, "terminalBench40": 0.0404040404040404, "terminalBench21": 0.737827715355805, "terminalbenchHard": 0.537878787878788, "scicode": 0.586805555555556, "hle": 0.470342910101946, "critpt": 0.177142857142857, "lcr": 0.82, "gpqa": 0.941414141414141, "omniscience": 31.883333333333333, "omniscienceAccuracy": 0.5485, "omniscienceNonHallucination": 0.4913252122554448, "gdpvalNormalized": 0.13799, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 112.986829907678, "medianTimeToFirstTokenSeconds": 25.866950722, "isOpenWeights": false }, { "name": "Qwen3 235B A22B (Reasoning)", "shortName": "Qwen3 235B", "slug": "qwen3-235b-a22b-instruct-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.49970850374162, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.1098, "critpt": 0, "lcr": 0, "gpqa": 0.7, "omniscience": -44.7, "omniscienceAccuracy": 0.18533333333333332, "omniscienceNonHallucination": 0.22381342062193121, "gdpvalNormalized": null, "price1mInputTokens": 0.7, "price1mOutputTokens": 8.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 60.6528077528649, "medianTimeToFirstTokenSeconds": 2.8327315445, "isOpenWeights": true }, { "name": "GPT-5 mini (Minimal)", "shortName": "GPT-5 mini (minimal)", "slug": "gpt-5-mini-minimal", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 9.90615279545732, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.143939393939394, "scicode": null, "hle": 0.0505097312326228, "critpt": 0, "lcr": 0.393333333333333, "gpqa": 0.686868686868687, "omniscience": -53.05, "omniscienceAccuracy": 0.19166666666666668, "omniscienceNonHallucination": 0.10659793814432994, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 2, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 110.649369538904, "medianTimeToFirstTokenSeconds": 0.899023223999961, "isOpenWeights": false }, { "name": "DeepSeek-V2.5 (Dec '24)", "shortName": "DeepSeek-V2.5 (Dec)", "slug": "deepseek-v2-5", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 6.61726180583266, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.423232323232323, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 2.0 Flash (Feb '25)", "shortName": "Gemini 2.0 Flash", "slug": "gemini-2-0-flash", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 8.94143668759525, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0426, "critpt": 0, "lcr": 0.313333333333333, "gpqa": 0.623232323232323, "omniscience": -42.53333333333333, "omniscienceAccuracy": 0.23466666666666666, "omniscienceNonHallucination": 0.1376306620209059, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "GPT-5.6 Terra (Max)", "shortName": "GPT-5.6 Terra (max)", "slug": "gpt-5-6-terra", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 42.0829248239494, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.3987235612691513, "terminalBench40": 0.353535353535354, "terminalBench21": 0.880149812734082, "terminalbenchHard": 0.575757575757576, "scicode": 0.549768518518518, "hle": 0.429101019462465, "critpt": 0.3, "lcr": 0.83, "gpqa": 0.925252525252525, "omniscience": 0.05, "omniscienceAccuracy": 0.468, "omniscienceNonHallucination": 0.12124060150375937, "gdpvalNormalized": 0.46613499999999997, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 92.6960038508539, "medianTimeToFirstTokenSeconds": 173.762642558, "isOpenWeights": false }, { "name": "Gemma 3n E4B Instruct Preview (May '25)", "shortName": "Gemma 3n E4B (May)", "slug": "gemma-3n-e4b-preview-0520", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 5.81096980860449, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0481927710843374, "critpt": null, "lcr": null, "gpqa": 0.277777777777778, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Kimi K2 Thinking", "shortName": "Kimi K2 Thinking", "slug": "kimi-k2-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Kimi", "intelligenceIndex": 22.0200270318461, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.310606060606061, "scicode": null, "hle": 0.238183503243744, "critpt": 0.0257142857142857, "lcr": 0.72, "gpqa": 0.838383838383838, "omniscience": -21.416666666666668, "omniscienceAccuracy": 0.30866666666666664, "omniscienceNonHallucination": 0.243731918997107, "gdpvalNormalized": null, "price1mInputTokens": 0.6, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.6, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 120.301720079914, "medianTimeToFirstTokenSeconds": 1.42231523650003, "isOpenWeights": true }, { "name": "MiMo-V2-Flash (Reasoning)", "shortName": "MiMo-V2-Flash", "slug": "mimo-v2-flash-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Xiaomi", "intelligenceIndex": 20.8177287591331, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.28030303030303, "scicode": null, "hle": 0.228452270620945, "critpt": 0.0428571428571429, "lcr": 0.706666666666667, "gpqa": 0.846464646464647, "omniscience": -45.35, "omniscienceAccuracy": 0.24833333333333332, "omniscienceNonHallucination": 0.0662971175166297, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "DeepSeek V4 Pro 0424 (Max)", "shortName": "DeepSeek V4 Pro (max)", "slug": "deepseek-v4-pro-0424", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 30.4496601315524, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.12152059767597723, "terminalBench40": 0.146464646464646, "terminalBench21": 0.640449438202247, "terminalbenchHard": 0.462121212121212, "scicode": 0.508101851851852, "hle": 0.375347544022243, "critpt": 0.128571428571429, "lcr": 0.746666666666667, "gpqa": 0.887878787878788, "omniscience": -10.733333333333333, "omniscienceAccuracy": 0.4295, "omniscienceNonHallucination": 0.05901256208004679, "gdpvalNormalized": 0.32218499999999994, "price1mInputTokens": 0.435, "price1mOutputTokens": 0.87, "cacheHitPrice": 0.0036, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 84.2248518150212, "medianTimeToFirstTokenSeconds": 1.56515483999999, "isOpenWeights": true }, { "name": "Qwen3 32B (Non-reasoning)", "shortName": "Qwen3 32B (non-reasoning)", "slug": "qwen3-32b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.3442784206229, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0413, "critpt": null, "lcr": 0, "gpqa": 0.535353535353535, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.16, "price1mOutputTokens": 0.64, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 106.273360080821, "medianTimeToFirstTokenSeconds": 2.56398231249998, "isOpenWeights": true }, { "name": "GPT-6 Luna (Low)", "shortName": "GPT-6 Luna (low)", "slug": "gpt-6-luna-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.9225480080866, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.0045215478273067, "terminalBench40": 0, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.46875, "hle": 0.202502316960148, "critpt": 0.0257142857142857, "lcr": 0.74, "gpqa": null, "omniscience": -9.166666666666666, "omniscienceAccuracy": 0.4076666666666667, "omniscienceNonHallucination": 0.15700619020821605, "gdpvalNormalized": 0.24613, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.01, "cacheWritePrice": 0.125, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 133.942959902865, "medianTimeToFirstTokenSeconds": 2.44879267350001, "isOpenWeights": false }, { "name": "Grok 4.6 (Low)", "shortName": "Grok 4.6 (low)", "slug": "grok-4-6-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 35.1246953482174, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.47528534374531, "terminalBench40": 0.0303030303030303, "terminalBench21": 0.752808988764045, "terminalbenchHard": null, "scicode": 0.494212962962963, "hle": 0.276181649675626, "critpt": 0.0571428571428571, "lcr": 0.806666666666667, "gpqa": 0.878787878787879, "omniscience": 25.9, "omniscienceAccuracy": 0.43283333333333335, "omniscienceNonHallucination": 0.6935057302380252, "gdpvalNormalized": 0.45121000000000006, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 64.6091530775868, "medianTimeToFirstTokenSeconds": 7.6351599555, "isOpenWeights": false }, { "name": "GPT-4o (Nov '24)", "shortName": "GPT-4o (Nov)", "slug": "gpt-4o", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 8.4416392330287, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0242, "critpt": 0, "lcr": 0.493333333333333, "gpqa": 0.543434343434343, "omniscience": -10.516666666666667, "omniscienceAccuracy": 0.19866666666666666, "omniscienceNonHallucination": 0.6208402662229617, "gdpvalNormalized": null, "price1mInputTokens": 2.5, "price1mOutputTokens": 10, "cacheHitPrice": 1.5, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 125.258109092846, "medianTimeToFirstTokenSeconds": 1.06414495300004, "isOpenWeights": false }, { "name": "G9v3-39A5B", "shortName": "G9v3-39A5B", "slug": "g9v3-39a5b", "isReasoning": true, "deprecated": false, "modelCreatorName": "AI9Stars", "intelligenceIndex": 21.8316172646825, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.325842696629214, "terminalbenchHard": null, "scicode": 0.368055555555556, "hle": 0.174698795180723, "critpt": 0.00285714285714286, "lcr": 0.653333333333333, "gpqa": 0.805050505050505, "omniscience": 3.7666666666666666, "omniscienceAccuracy": 0.14866666666666667, "omniscienceNonHallucination": 0.8696162881754111, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "GPT-5.4 nano (Non-reasoning)", "shortName": "GPT-5.4 nano (non-reasoning)", "slug": "gpt-5-4-nano-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 11.6892323777027, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.242424242424242, "scicode": null, "hle": 0.0407784986098239, "critpt": 0, "lcr": 0.296666666666667, "gpqa": 0.557575757575758, "omniscience": -37.483333333333334, "omniscienceAccuracy": 0.14883333333333335, "omniscienceNonHallucination": 0.3847660074407676, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.25, "cacheHitPrice": 0.02, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 157.313918899053, "medianTimeToFirstTokenSeconds": 0.764928419500101, "isOpenWeights": false }, { "name": "Claude 4.5 Haiku (Non-reasoning)", "shortName": "Claude 4.5 Haiku (non-reasoning)", "slug": "claude-4-5-haiku", "isReasoning": false, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 15.410949885381, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.272727272727273, "scicode": null, "hle": 0.0421686746987952, "critpt": 0, "lcr": 0.496666666666667, "gpqa": 0.646464646464646, "omniscience": -7.566666666666666, "omniscienceAccuracy": 0.14433333333333334, "omniscienceNonHallucination": 0.7428905336969225, "gdpvalNormalized": null, "price1mInputTokens": 1, "price1mOutputTokens": 5, "cacheHitPrice": 0.1, "cacheWritePrice": 1.25, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 83.0495040157319, "medianTimeToFirstTokenSeconds": 0.621509999999944, "isOpenWeights": false }, { "name": "GPT-5 mini (Medium)", "shortName": "GPT-5 mini (medium)", "slug": "gpt-5-mini-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.5991778305353, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.287878787878788, "scicode": null, "hle": 0.159406858202039, "critpt": 0.0142857142857143, "lcr": 0.723333333333333, "gpqa": 0.803030303030303, "omniscience": -11.033333333333333, "omniscienceAccuracy": 0.23066666666666666, "omniscienceNonHallucination": 0.5567590987868285, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 2, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 110.163452038506, "medianTimeToFirstTokenSeconds": 13.172368481, "isOpenWeights": false }, { "name": "GPT-5 (Minimal)", "shortName": "GPT-5 (minimal)", "slug": "gpt-5-minimal", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 11.4481891190946, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.181818181818182, "scicode": null, "hle": 0.0602409638554217, "critpt": 0, "lcr": null, "gpqa": 0.672727272727273, "omniscience": -33.8, "omniscienceAccuracy": 0.29633333333333334, "omniscienceNonHallucination": 0.09853150165798197, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 65.5451462044603, "medianTimeToFirstTokenSeconds": 1.294201214, "isOpenWeights": false }, { "name": "DiffusionGemma 26B A4B", "shortName": "DiffusionGemma 26B A4B", "slug": "diffusiongemma-26b-a4b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 9.51014305662543, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.123595505617978, "terminalbenchHard": null, "scicode": null, "hle": 0.108433734939759, "critpt": 0.00285714285714286, "lcr": 0.196666666666667, "gpqa": 0.668686868686869, "omniscience": -59.233333333333334, "omniscienceAccuracy": 0.16583333333333333, "omniscienceNonHallucination": 0.09110889110889109, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GPT-5.5 Instant (June 2026)", "shortName": "GPT-5.5 Instant (June 2026)", "slug": "gpt-5-5-instant-06-26", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 26.0135173401368, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.6914853678439488, "terminalBench40": 0.126262626262626, "terminalBench21": 0.348314606741573, "terminalbenchHard": null, "scicode": 0.525462962962963, "hle": 0.199258572752549, "critpt": 0, "lcr": 0.7, "gpqa": 0.823232323232323, "omniscience": 3.7, "omniscienceAccuracy": 0.44416666666666665, "omniscienceNonHallucination": 0.2674662668665667, "gdpvalNormalized": 0, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 110.830418759174, "medianTimeToFirstTokenSeconds": 1.05841464099996, "isOpenWeights": false }, { "name": "Qwen Chat 14B", "shortName": "Qwen Chat 14B", "slug": "qwen-chat-14b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.03498213171439, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "Llama 3.3 Nemotron Super 49B v1 (Non-reasoning)", "shortName": "Llama 3.3 Nemotron Super 49B (non-reasoning)", "slug": "llama-3-3-nemotron-super-49b", "isReasoning": false, "deprecated": true, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.27513732255953, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0381, "critpt": 0, "lcr": 0.12, "gpqa": 0.517171717171717, "omniscience": -50.3, "omniscienceAccuracy": 0.13466666666666666, "omniscienceNonHallucination": 0.2630970724191063, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "GPT-6 Sol (Max)", "shortName": "GPT-6 Sol (max)", "slug": "gpt-6-sol", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 47.5276426437724, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.049410804880785, "terminalBench40": 0.439393939393939, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.576388888888889, "hle": 0.479147358665431, "critpt": 0.308571428571429, "lcr": 0.836666666666667, "gpqa": null, "omniscience": 27.116666666666667, "omniscienceAccuracy": 0.5448333333333333, "omniscienceNonHallucination": 0.3987550347857928, "gdpvalNormalized": 0.49346500000000004, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 872000, "medianOutputTokensPerSecond": 74.4011617229816, "medianTimeToFirstTokenSeconds": 179.052404721, "isOpenWeights": false }, { "name": "Gemini 2.0 Pro Experimental (Feb '25)", "shortName": "Gemini 2.0 Pro Experimental", "slug": "gemini-2-0-pro-experimental-02-05", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 8.7462081604643, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0619, "critpt": null, "lcr": null, "gpqa": 0.622222222222222, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "Grok 4.20 0309 v2 (Reasoning)", "shortName": "Grok 4.20 0309 v2", "slug": "grok-4-20", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 25.6550155187053, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.378787878787879, "scicode": null, "hle": 0.345227062094532, "critpt": 0.0657142857142857, "lcr": 0.69, "gpqa": 0.911111111111111, "omniscience": 14.833333333333334, "omniscienceAccuracy": 0.2748333333333333, "omniscienceNonHallucination": 0.8255573431395081, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 2000000, "medianOutputTokensPerSecond": 101.595917485303, "medianTimeToFirstTokenSeconds": 20.123797251, "isOpenWeights": false }, { "name": "DeepSeek Coder V2 Lite Instruct", "shortName": "DeepSeek Coder V2 Lite", "slug": "deepseek-coder-v2-lite", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 5.33591177993782, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0536, "critpt": null, "lcr": null, "gpqa": 0.319191919191919, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 2.5 Flash Preview (Sep '25) (Reasoning)", "shortName": "Gemini 2.5 Flash (Sep)", "slug": "gemini-2-5-flash-preview-09-2025-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 15.4671894654212, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.166666666666667, "scicode": null, "hle": 0.138090824837813, "critpt": 0.00285714285714286, "lcr": 0.71, "gpqa": 0.792929292929293, "omniscience": -36.1, "omniscienceAccuracy": 0.2833333333333333, "omniscienceNonHallucination": 0.10093023255813949, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Falcon-H1R-7B", "shortName": "Falcon-H1R-7B", "slug": "falcon-h1r-7b", "isReasoning": true, "deprecated": false, "modelCreatorName": "TII UAE", "intelligenceIndex": 7.82776928077145, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.10982391102873, "critpt": 0.00285714285714286, "lcr": 0.11, "gpqa": 0.660606060606061, "omniscience": -61.3, "omniscienceAccuracy": 0.1455, "omniscienceNonHallucination": 0.11234640140433005, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GPT-5.5 (Low)", "shortName": "GPT-5.5 (low)", "slug": "gpt-5-5-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 30.709518471208, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.655430711610487, "terminalbenchHard": 0.522727272727273, "scicode": null, "hle": 0.326691380908248, "critpt": 0.08, "lcr": 0.81, "gpqa": 0.91010101010101, "omniscience": 15.1, "omniscienceAccuracy": 0.5485, "omniscienceNonHallucination": 0.11960132890365449, "gdpvalNormalized": 0.26605999999999996, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 922000, "medianOutputTokensPerSecond": 83.3595810662624, "medianTimeToFirstTokenSeconds": 1.64262621400002, "isOpenWeights": false }, { "name": "Step 3.5 Flash", "shortName": "Step 3.5 Flash", "slug": "step-3-5-flash-0202", "isReasoning": true, "deprecated": true, "modelCreatorName": "StepFun", "intelligenceIndex": 16.6169431254306, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.272727272727273, "scicode": null, "hle": 0.211306765523633, "critpt": 0.0246938775510204, "lcr": 0.5, "gpqa": 0.831313131313131, "omniscience": -41.81666666666667, "omniscienceAccuracy": 0.23616666666666666, "omniscienceNonHallucination": 0.14335588042766745, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.3, "cacheHitPrice": 0.02, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 145.257767200765, "medianTimeToFirstTokenSeconds": 3.38954943399995, "isOpenWeights": true }, { "name": "GLM-4.7-Flash (Non-reasoning)", "shortName": "GLM-4.7-Flash (non-reasoning)", "slug": "glm-4-7-flash-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 10.555698892906, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0495829471733086, "critpt": 0, "lcr": 0.203333333333333, "gpqa": 0.451515151515151, "omniscience": -68.93333333333334, "omniscienceAccuracy": 0.1305, "omniscienceNonHallucination": 0.057120950737971965, "gdpvalNormalized": null, "price1mInputTokens": 0.07, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.04, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 80.748772852629, "medianTimeToFirstTokenSeconds": 1.70960667199994, "isOpenWeights": true }, { "name": "Olmo 3 32B Think", "shortName": "Olmo 3 32B Think", "slug": "olmo-3-32b-think", "isReasoning": true, "deprecated": true, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 6.47275394213019, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0644114921223355, "critpt": 0, "lcr": 0, "gpqa": 0.61010101010101, "omniscience": -58.06666666666667, "omniscienceAccuracy": 0.14866666666666667, "omniscienceNonHallucination": 0.14330462020360224, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": true }, { "name": "Qwen3 Next 80B A3B (Reasoning)", "shortName": "Qwen3 Next 80B A3B", "slug": "qwen3-next-80b-a3b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 11.1973684395653, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0674157303370786, "terminalbenchHard": 0.0984848484848485, "scicode": null, "hle": 0.125579240037071, "critpt": 0, "lcr": 0.636666666666667, "gpqa": 0.758585858585858, "omniscience": -51.416666666666664, "omniscienceAccuracy": 0.19033333333333333, "omniscienceNonHallucination": 0.12988884314532734, "gdpvalNormalized": 0, "price1mInputTokens": 0.15, "price1mOutputTokens": 1.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 157.009273680431, "medianTimeToFirstTokenSeconds": 2.35877552950001, "isOpenWeights": true }, { "name": "Grok 4.6 (High)", "shortName": "Grok 4.6 (high)", "slug": "grok-4-6", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 44.3113073012592, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.8589423352127492, "terminalBench40": 0.212121212121212, "terminalBench21": 0.883895131086142, "terminalbenchHard": null, "scicode": 0.564814814814815, "hle": 0.429101019462465, "critpt": 0.171428571428571, "lcr": 0.803333333333333, "gpqa": 0.94949494949495, "omniscience": 30.483333333333334, "omniscienceAccuracy": 0.48233333333333334, "omniscienceNonHallucination": 0.6571152607855764, "gdpvalNormalized": 0.55452, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 64.1727438010873, "medianTimeToFirstTokenSeconds": 35.8974612315, "isOpenWeights": false }, { "name": "gpt-oss-120b (Low)", "shortName": "gpt-oss-120b (low)", "slug": "gpt-oss-120b-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 10.2105650347196, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.138576779026217, "terminalbenchHard": 0.053030303030303, "scicode": null, "hle": 0.0588507877664504, "critpt": 0, "lcr": 0.46, "gpqa": 0.671717171717172, "omniscience": -53.5, "omniscienceAccuracy": 0.198, "omniscienceNonHallucination": 0.08603491271820451, "gdpvalNormalized": 0, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.5449999999999999, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 170.596021589476, "medianTimeToFirstTokenSeconds": 0.866880757999979, "isOpenWeights": true }, { "name": "o3-pro", "shortName": "o3-pro", "slug": "o3-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 21.869975912052, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.845454545454545, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 20, "price1mOutputTokens": 80, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Mixtral 8x22B Instruct", "shortName": "Mixtral 8x22B", "slug": "mistral-8x22b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 5.74434049511794, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0401, "critpt": null, "lcr": null, "gpqa": 0.332323232323232, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65384, "isOpenWeights": true }, { "name": "Qwen3 Max Thinking (Preview)", "shortName": "Qwen3 Max Thinking (Preview)", "slug": "qwen3-max-thinking-preview", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 16.290984641221, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.174242424242424, "scicode": null, "hle": 0.126506024096386, "critpt": 0, "lcr": 0.62, "gpqa": 0.775757575757576, "omniscience": -39.11666666666667, "omniscienceAccuracy": 0.2735, "omniscienceNonHallucination": 0.08511126405138791, "gdpvalNormalized": null, "price1mInputTokens": 1.2, "price1mOutputTokens": 6, "cacheHitPrice": 0.24, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 52.6650228506983, "medianTimeToFirstTokenSeconds": 4.17780434350004, "isOpenWeights": false }, { "name": "GLM-5.3 (Low)", "shortName": "GLM-5.3 (low)", "slug": "glm-5-3-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Z AI", "intelligenceIndex": 34.2990078159643, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.8522174973386498, "terminalBench40": 0.348484848484849, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.420138888888889, "hle": 0.365616311399444, "critpt": 0.145714285714286, "lcr": 0.726666666666667, "gpqa": null, "omniscience": -8.4, "omniscienceAccuracy": 0.33916666666666667, "omniscienceNonHallucination": 0.3596469104665826, "gdpvalNormalized": 0.40241499999999997, "price1mInputTokens": 1.4, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.26, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 68.9364006955905, "medianTimeToFirstTokenSeconds": 3.1406351045, "isOpenWeights": true }, { "name": "Devstral Small (May '25)", "shortName": "Devstral Small (May)", "slug": "devstral-small-2505", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 8.73503199772396, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0606060606060606, "scicode": null, "hle": 0.0403151065801668, "critpt": 0, "lcr": 0.32, "gpqa": 0.434343434343434, "omniscience": -56.78333333333333, "omniscienceAccuracy": 0.15866666666666668, "omniscienceNonHallucination": 0.13648969889064977, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GPT-5.6 Terra (Low)", "shortName": "GPT-5.6 Terra (low)", "slug": "gpt-5-6-terra-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 27.496203401716, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.1444617614290821, "terminalBench40": 0.0151515151515152, "terminalBench21": 0.625468164794007, "terminalbenchHard": 0.439393939393939, "scicode": 0.498842592592593, "hle": 0.291936978683967, "critpt": 0.0942857142857143, "lcr": 0.713333333333333, "gpqa": 0.843434343434343, "omniscience": -6.833333333333333, "omniscienceAccuracy": 0.43733333333333335, "omniscienceNonHallucination": 0.101303317535545, "gdpvalNormalized": 0.29802999999999996, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 73.350801384417, "medianTimeToFirstTokenSeconds": 1.8604572975, "isOpenWeights": false }, { "name": "Claude 4.1 Opus (Reasoning)", "shortName": "Claude 4.1 Opus", "slug": "claude-4-1-opus-thinking", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 22.8471944399332, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.343352330715909, "scicode": null, "hle": 0.124652455977757, "critpt": 0, "lcr": 0.76, "gpqa": 0.809, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 15, "price1mOutputTokens": 75, "cacheHitPrice": 1.5, "cacheWritePrice": 18.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Gemma 4 31B (Reasoning)", "shortName": "Gemma 4 31B", "slug": "gemma-4-31b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 14.6689500745828, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0, "terminalBench40": 0, "terminalBench21": 0.434456928838951, "terminalbenchHard": 0.363636363636364, "scicode": 0.454861111111111, "hle": 0.236329935125116, "critpt": 0.0142857142857143, "lcr": 0.696666666666667, "gpqa": 0.856565656565657, "omniscience": -47.93333333333333, "omniscienceAccuracy": 0.20033333333333334, "omniscienceNonHallucination": 0.1500625260525219, "gdpvalNormalized": 0.05692500000000001, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 35.5690160624163, "medianTimeToFirstTokenSeconds": 1.06343058699986, "isOpenWeights": true }, { "name": "Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)", "shortName": "Claude Fable 5 (with fallback)", "slug": "claude-fable-5", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 49.6258202733378, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 8.745961651687406, "terminalBench40": 0.424242424242424, "terminalBench21": 0.846441947565543, "terminalbenchHard": 0.628787878787879, "scicode": 0.609953703703704, "hle": 0.554680259499537, "critpt": 0.2857142857, "lcr": 0.823333333333333, "gpqa": 0.926262626262626, "omniscience": 43.3, "omniscienceAccuracy": 0.6535, "omniscienceNonHallucination": 0.36363636363636365, "gdpvalNormalized": 0.547705, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 1, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 64.2139353927659, "medianTimeToFirstTokenSeconds": 98.420505393, "isOpenWeights": false }, { "name": "GLM-5.3 (Max)", "shortName": "GLM-5.3 (max)", "slug": "glm-5-3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Z AI", "intelligenceIndex": 44.777392385614, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.0056375150449584, "terminalBench40": 0.419191919191919, "terminalBench21": 0.838951310861423, "terminalbenchHard": null, "scicode": 0.590277777777778, "hle": 0.422613531047266, "critpt": 0.191428571428571, "lcr": 0.796666666666667, "gpqa": 0.917171717171717, "omniscience": 14.3, "omniscienceAccuracy": 0.3385, "omniscienceNonHallucination": 0.7044595616024187, "gdpvalNormalized": 0.5718099999999999, "price1mInputTokens": 1.4, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.26, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 67.8478068657519, "medianTimeToFirstTokenSeconds": 3.250817074, "isOpenWeights": true }, { "name": "Qwen3 VL 30B A3B (Reasoning)", "shortName": "Qwen3 VL 30B A3B", "slug": "qwen3-vl-30b-a3b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.45305611694923, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.053030303030303, "scicode": null, "hle": 0.0894346617238184, "critpt": 0, "lcr": null, "gpqa": 0.72020202020202, "omniscience": -57.63333333333333, "omniscienceAccuracy": 0.16933333333333334, "omniscienceNonHallucination": 0.1023274478330658, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 2.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 112.245484904355, "medianTimeToFirstTokenSeconds": 2.17924950600001, "isOpenWeights": true }, { "name": "Devstral Small 2", "shortName": "Devstral Small 2", "slug": "devstral-small-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 7.5369358990318, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.295880149812734, "terminalbenchHard": 0.166666666666667, "scicode": 0.324074074074074, "hle": 0.0347544022242817, "critpt": 0, "lcr": 0.28, "gpqa": 0.532323232323232, "omniscience": -57.7, "omniscienceAccuracy": 0.158, "omniscienceNonHallucination": 0.12707838479809974, "gdpvalNormalized": 0.007860000000000013, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "QwQ 32B", "shortName": "QwQ-32B", "slug": "qwq-32b", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.47210556751002, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0735, "critpt": null, "lcr": 0.266666666666667, "gpqa": 0.592929292929293, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.66, "price1mOutputTokens": 1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Llama 3.1 Nemotron Ultra 253B v1 (Reasoning)", "shortName": "Llama Nemotron Ultra", "slug": "llama-3-1-nemotron-ultra-253b-v1-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.52746905624501, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0738, "critpt": 0, "lcr": null, "gpqa": 0.728282828282828, "omniscience": -44.86666666666667, "omniscienceAccuracy": 0.20066666666666666, "omniscienceNonHallucination": 0.18765638031693077, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Claude 4 Sonnet (Non-reasoning)", "shortName": "Claude 4 Sonnet (non-reasoning)", "slug": "claude-4-sonnet", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 16.6124406514554, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.272727272727273, "scicode": null, "hle": 0.0426320667284523, "critpt": 0.0114285714285714, "lcr": 0.44, "gpqa": 0.682828282828283, "omniscience": -9.016666666666667, "omniscienceAccuracy": 0.22666666666666666, "omniscienceNonHallucination": 0.590301724137931, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Gemma 4 31B (Non-reasoning)", "shortName": "Gemma 4 31B (non-reasoning)", "slug": "gemma-4-31b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 13.9198819005287, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.292134831460674, "terminalbenchHard": 0.303030303030303, "scicode": null, "hle": 0.118164967562558, "critpt": 0, "lcr": 0.466666666666667, "gpqa": 0.762626262626263, "omniscience": -51.68333333333333, "omniscienceAccuracy": 0.16633333333333333, "omniscienceNonHallucination": 0.1805277888844462, "gdpvalNormalized": 0.01764499999999998, "price1mInputTokens": 0.14, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.14, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 40.3201822103343, "medianTimeToFirstTokenSeconds": 2.00712007149991, "isOpenWeights": true }, { "name": "GPT-5.1 (Non-reasoning)", "shortName": "GPT-5.1 (non-reasoning)", "slug": "gpt-5-1-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 13.3137294472025, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.227272727272727, "scicode": null, "hle": 0.0528266913809083, "critpt": 0, "lcr": 0.45, "gpqa": 0.643434343434343, "omniscience": -34.45, "omniscienceAccuracy": 0.2946666666666667, "omniscienceNonHallucination": 0.0938090737240076, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 109.191187735769, "medianTimeToFirstTokenSeconds": 1.027473628, "isOpenWeights": false }, { "name": "Gemini 3 Pro Preview (High)", "shortName": "Gemini 3 Pro Preview (high)", "slug": "gemini-3-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 27.9579716203661, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.416666666666667, "scicode": null, "hle": 0.397126969416126, "critpt": 0.0914285714285714, "lcr": 0.76, "gpqa": 0.908080808080808, "omniscience": 15.283333333333333, "omniscienceAccuracy": 0.5575, "omniscienceNonHallucination": 0.08549905838041427, "gdpvalNormalized": null, "price1mInputTokens": 2, "price1mOutputTokens": 12, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Gemma 3 1B Instruct", "shortName": "Gemma 3 1B", "slug": "gemma-3-1b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 4.83423948, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0532900834105653, "critpt": 0, "lcr": 0, "gpqa": 0.237373737373737, "omniscience": -75.45, "omniscienceAccuracy": 0.0375, "omniscienceNonHallucination": 0.17714285714285716, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Qwen1.5 Chat 110B", "shortName": "Qwen1.5 Chat 110B", "slug": "qwen1.5-110b-chat", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.65355605607575, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.288888888888889, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "GLM-4.6V (Reasoning)", "shortName": "GLM-4.6V", "slug": "glm-4-6v-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 11.2162066974921, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.143939393939394, "scicode": null, "hle": 0.0963855421686747, "critpt": 0, "lcr": 0.486666666666667, "gpqa": 0.719191919191919, "omniscience": -26.95, "omniscienceAccuracy": 0.16183333333333333, "omniscienceNonHallucination": 0.4853847683436071, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 0.9, "cacheHitPrice": 0.1775, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 80.0024179897852, "medianTimeToFirstTokenSeconds": 3.17496136699998, "isOpenWeights": true }, { "name": "MiniCPM5-1B (Non-reasoning)", "shortName": "MiniCPM5-1B (non-reasoning)", "slug": "minicpm5-1b-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenBMB", "intelligenceIndex": 8.69473818567247, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0454124189063948, "critpt": 0, "lcr": 0.05, "gpqa": 0.268686868686869, "omniscience": -0.8, "omniscienceAccuracy": 0.001, "omniscienceNonHallucination": 0.990990990990991, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 4 Argon (High)", "shortName": "Gemini 4 Argon (high)", "slug": "gemini-4-argon", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 52.5605655745982, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.990322379299008, "terminalBench40": 0.570707070707071, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.618055555555556, "hle": 0.570898980537535, "critpt": 0.271428571428571, "lcr": 0.796666666666667, "gpqa": null, "omniscience": 42.35, "omniscienceAccuracy": 0.499, "omniscienceNonHallucination": 0.8493013972055888, "gdpvalNormalized": 0.555665, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.1, "cacheWritePrice": 2, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Qwen3 VL 32B (Reasoning)", "shortName": "Qwen3 VL 32B", "slug": "qwen3-vl-32b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 11.8701729641937, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0757575757575758, "scicode": null, "hle": 0.101482854494903, "critpt": 0, "lcr": 0.553333333333333, "gpqa": 0.733333333333333, "omniscience": -52.516666666666666, "omniscienceAccuracy": 0.16966666666666666, "omniscienceNonHallucination": 0.16318747490967478, "gdpvalNormalized": null, "price1mInputTokens": 0.16, "price1mOutputTokens": 0.64, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 89.9456877267092, "medianTimeToFirstTokenSeconds": 2.81062546050009, "isOpenWeights": true }, { "name": "Hermes 4 - Llama-3.1 405B (Non-reasoning)", "shortName": "Hermes 4 405B (non-reasoning)", "slug": "hermes-4-llama-3-1-405b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Nous Research", "intelligenceIndex": 7.41221110231024, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0984848484848485, "scicode": null, "hle": 0.0421686746987952, "critpt": 0, "lcr": 0.22, "gpqa": 0.536363636363636, "omniscience": -33.28333333333333, "omniscienceAccuracy": 0.2605, "omniscienceNonHallucination": 0.19765607392382245, "gdpvalNormalized": null, "price1mInputTokens": 1, "price1mOutputTokens": 3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 39.0580823482533, "medianTimeToFirstTokenSeconds": 2.38459318450001, "isOpenWeights": true }, { "name": "GPT-6 Sol (Non-reasoning)", "shortName": "GPT-6 Sol (non-reasoning)", "slug": "gpt-6-sol-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 28.0921663410513, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.333379364253275, "terminalBench40": 0.131313131313131, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.47337962962963, "hle": 0.183966635773865, "critpt": 0.04, "lcr": 0.64, "gpqa": null, "omniscience": -0.85, "omniscienceAccuracy": 0.4518333333333333, "omniscienceNonHallucination": 0.16023107327455155, "gdpvalNormalized": 0.36269500000000005, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 872000, "medianOutputTokensPerSecond": 64.4842877462774, "medianTimeToFirstTokenSeconds": 0.978946806999943, "isOpenWeights": false }, { "name": "Granite 4.1 30B", "shortName": "Granite 4.1 30B", "slug": "granite-4-1-30b", "isReasoning": false, "deprecated": true, "modelCreatorName": "IBM", "intelligenceIndex": 7.43693799607016, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0262172284644195, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.041241890639481, "critpt": 0, "lcr": 0.236666666666667, "gpqa": 0.480808080808081, "omniscience": -67.78333333333333, "omniscienceAccuracy": 0.1395, "omniscienceNonHallucination": 0.05016463296533025, "gdpvalNormalized": 0, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "DeepSeek V4 Pro 0813 (Non-reasoning)", "shortName": "DeepSeek V4 Pro 0813 (non-reasoning)", "slug": "deepseek-v4-pro-non-reasoning", "isReasoning": false, "deprecated": false, "modelCreatorName": "DeepSeek", "intelligenceIndex": 20.3605101726847, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.47742401619163854, "terminalBench40": 0.0151515151515152, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.399305555555556, "hle": 0.105653382761817, "critpt": 0, "lcr": 0.506666666666667, "gpqa": null, "omniscience": -12.383333333333333, "omniscienceAccuracy": 0.4145, "omniscienceNonHallucination": 0.080557927697125, "gdpvalNormalized": 0.27621, "price1mInputTokens": 1.32, "price1mOutputTokens": 3.96, "cacheHitPrice": 0.044, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 179.697129903173, "medianTimeToFirstTokenSeconds": 2.06404605699999, "isOpenWeights": true }, { "name": "GPT-5.4 nano (Xhigh)", "shortName": "GPT-5.4 nano (xhigh)", "slug": "gpt-5-4-nano", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.7197352504557, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.1830802502922872, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.606741573033708, "terminalbenchHard": 0.424242424242424, "scicode": 0.472222222222222, "hle": 0.282669138090825, "critpt": 0.0925306122448979, "lcr": 0.766666666666667, "gpqa": 0.817171717171717, "omniscience": -29.466666666666665, "omniscienceAccuracy": 0.25666666666666665, "omniscienceNonHallucination": 0.25829596412556055, "gdpvalNormalized": 0.218325, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.25, "cacheHitPrice": 0.02, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 160.181289719855, "medianTimeToFirstTokenSeconds": 112.2932968835, "isOpenWeights": false }, { "name": "Muse Spark 1.3 (Xhigh)", "shortName": "Muse Spark 1.3 (xhigh)", "slug": "muse-spark-1-3-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "Meta", "intelligenceIndex": 45.0732966922226, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.3677936514692752, "terminalBench40": 0.166666666666667, "terminalBench21": 0.853932584269663, "terminalbenchHard": null, "scicode": 0.597222222222222, "hle": 0.47451343836886, "critpt": 0.26, "lcr": 0.83, "gpqa": 0.941414141414141, "omniscience": 23.116666666666667, "omniscienceAccuracy": 0.4151666666666667, "omniscienceNonHallucination": 0.6853804502707324, "gdpvalNormalized": 0.55932, "price1mInputTokens": 1.25, "price1mOutputTokens": 4.25, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 141.660302159044, "medianTimeToFirstTokenSeconds": 30.54504788, "isOpenWeights": false }, { "name": "Claude Sonnet 5 (Non-reasoning, High Effort)", "shortName": "Claude Sonnet 5 (non-reasoning)", "slug": "claude-sonnet-5-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 23.2, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.752808988764045, "terminalbenchHard": null, "scicode": null, "hle": 0.190454124189064, "critpt": 0.0114285714285714, "lcr": 0.7, "gpqa": 0.8, "omniscience": -0.65, "omniscienceAccuracy": 0.338, "omniscienceNonHallucination": 0.479607250755287, "gdpvalNormalized": 0.35005, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 59.5246787189795, "medianTimeToFirstTokenSeconds": 1.08900155000003, "isOpenWeights": false }, { "name": "Qwen3 VL 235B A22B Instruct", "shortName": "Qwen3 VL 235B A22B", "slug": "qwen3-vl-235b-a22b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.93699415502801, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0662650602409639, "critpt": 0, "lcr": 0.326666666666667, "gpqa": 0.712121212121212, "omniscience": -52.666666666666664, "omniscienceAccuracy": 0.203, "omniscienceNonHallucination": 0.08448347971560022, "gdpvalNormalized": null, "price1mInputTokens": 0.4, "price1mOutputTokens": 1.6, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 52.3699289010586, "medianTimeToFirstTokenSeconds": 2.6674216815, "isOpenWeights": true }, { "name": "Muse Spark 1.1 (Xhigh)", "shortName": "Muse Spark 1.1 (xhigh)", "slug": "muse-spark-1-1", "isReasoning": true, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 33.7298425919846, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.3802453886670354, "terminalBench40": 0.0606060606060606, "terminalBench21": 0.779026217228464, "terminalbenchHard": null, "scicode": 0.587962962962963, "hle": 0.462001853568119, "critpt": 0.151428571428571, "lcr": 0.776666666666667, "gpqa": 0.897979797979798, "omniscience": 28.083333333333332, "omniscienceAccuracy": 0.5205, "omniscienceNonHallucination": 0.500173792144595, "gdpvalNormalized": 0.35417999999999994, "price1mInputTokens": 1.25, "price1mOutputTokens": 4.25, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 1048576, "isOpenWeights": false }, { "name": "DeepSeek V3.2 (Reasoning)", "shortName": "DeepSeek V3.2", "slug": "deepseek-v3-2-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 21.4867901704834, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.468164794007491, "terminalbenchHard": 0.356060606060606, "scicode": null, "hle": 0.245597775718258, "critpt": 0.0285714285714286, "lcr": 0.733333333333333, "gpqa": 0.84040404040404, "omniscience": -22.483333333333334, "omniscienceAccuracy": 0.32966666666666666, "omniscienceNonHallucination": 0.1727996021879662, "gdpvalNormalized": 0.09041000000000002, "price1mInputTokens": 0.28, "price1mOutputTokens": 0.42, "cacheHitPrice": 0.014, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Grok 4.6 (Xhigh)", "shortName": "Grok 4.6 (xhigh)", "slug": "grok-4-6-xhigh", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 44.199810436996, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.323692124283027, "terminalBench40": 0.171717171717172, "terminalBench21": 0.880149812734082, "terminalbenchHard": null, "scicode": 0.530092592592593, "hle": 0.440685820203893, "critpt": 0.197142857142857, "lcr": 0.81, "gpqa": 0.935353535353535, "omniscience": 29.316666666666666, "omniscienceAccuracy": 0.43, "omniscienceNonHallucination": 0.7599415204678363, "gdpvalNormalized": 0.56611, "price1mInputTokens": 2, "price1mOutputTokens": 6, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 500000, "medianOutputTokensPerSecond": 64.8515508106648, "medianTimeToFirstTokenSeconds": 31.3291955105, "isOpenWeights": false }, { "name": "Exaone 4.0 1.2B (Reasoning)", "shortName": "Exaone 4.0 1.2B", "slug": "exaone-4-0-1-2b-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "LG AI Research", "intelligenceIndex": 5.27138994054015, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0597775718257646, "critpt": 0, "lcr": 0, "gpqa": 0.515151515151515, "omniscience": -79.81666666666666, "omniscienceAccuracy": 0.06833333333333333, "omniscienceNonHallucination": 0.06994633273703044, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 64000, "isOpenWeights": true }, { "name": "Gemma 4 E4B (Reasoning)", "shortName": "Gemma 4 E4B", "slug": "gemma-4-e4b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 8.91477392266452, "intelligenceIndexIsEstimated": true, "terminalBench40": 0, "terminalBench21": 0.0187265917602996, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0375347544022243, "critpt": 0.00571428571428571, "lcr": 0.32, "gpqa": 0.575757575757576, "omniscience": -19.7, "omniscienceAccuracy": 0.08583333333333333, "omniscienceNonHallucination": 0.6906107566089335, "gdpvalNormalized": 0, "price1mInputTokens": 0.02, "price1mOutputTokens": 0.1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 69.5209106642696, "medianTimeToFirstTokenSeconds": 0.844578773999984, "isOpenWeights": true }, { "name": "DeepSeek V4 Flash 0420 (Max)", "shortName": "DeepSeek V4 Flash (max)", "slug": "deepseek-v4-flash-0420", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 24.1673138235549, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.1074631228275786, "terminalBench40": 0.0252525252525253, "terminalBench21": 0.617977528089888, "terminalbenchHard": 0.356060606060606, "scicode": 0.452546296296296, "hle": 0.348470806302132, "critpt": 0.0714285714285714, "lcr": 0.743333333333333, "gpqa": 0.893939393939394, "omniscience": -23.9, "omniscienceAccuracy": 0.36816666666666664, "omniscienceNonHallucination": 0.039039831179108364, "gdpvalNormalized": 0.2615, "price1mInputTokens": 0.13, "price1mOutputTokens": 0.28, "cacheHitPrice": 0.028, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": true }, { "name": "Claude 3.7 Sonnet (Non-reasoning)", "shortName": "Claude 3.7 Sonnet (non-reasoning)", "slug": "claude-3-7-sonnet", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 15.2705782730385, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.212121212121212, "scicode": null, "hle": 0.0416, "critpt": 0, "lcr": 0.516666666666667, "gpqa": 0.655555555555556, "omniscience": -9.716666666666667, "omniscienceAccuracy": 0.282, "omniscienceNonHallucination": 0.4719127205199629, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "GPT-5.6 Sol (Max)", "shortName": "GPT-5.6 Sol (max)", "slug": "gpt-5-6-sol", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 46.9727068915036, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.98845538085375, "terminalBench40": 0.398989898989899, "terminalBench21": 0.880149812734082, "terminalbenchHard": 0.659090909090909, "scicode": 0.570601851851852, "hle": 0.494902687673772, "critpt": 0.322857142857143, "lcr": 0.84, "gpqa": 0.941414141414141, "omniscience": 21.966666666666665, "omniscienceAccuracy": 0.594, "omniscienceNonHallucination": 0.07799671592775037, "gdpvalNormalized": 0.54395, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.4, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 79.5998245212136, "medianTimeToFirstTokenSeconds": 106.1023721615, "isOpenWeights": false }, { "name": "Solar Mini", "shortName": "Solar Mini", "slug": "solar-mini", "isReasoning": false, "deprecated": true, "modelCreatorName": "Upstage", "intelligenceIndex": 6.4088294637731, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.15, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "GPT-5.6 Sol (Xhigh)", "shortName": "GPT-5.6 Sol (xhigh)", "slug": "gpt-5-6-sol-xhigh", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 44.008876369169, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.1843629381907275, "terminalBench40": 0.247474747474747, "terminalBench21": 0.895131086142322, "terminalbenchHard": 0.613636363636364, "scicode": 0.570601851851852, "hle": 0.473123262279889, "critpt": 0.285714285714286, "lcr": 0.823333333333333, "gpqa": 0.931313131313131, "omniscience": 20.983333333333334, "omniscienceAccuracy": 0.5881666666666666, "omniscienceNonHallucination": 0.08134358559287735, "gdpvalNormalized": 0.52378, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.4, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 77.5583922217525, "medianTimeToFirstTokenSeconds": 41.7884933474999, "isOpenWeights": false }, { "name": "GPT-6 Sol (Xhigh)", "shortName": "GPT-6 Sol (xhigh)", "slug": "gpt-6-sol-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 44.1009705516407, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.523567515784736, "terminalBench40": 0.303030303030303, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.550925925925926, "hle": 0.462928637627433, "critpt": 0.28, "lcr": 0.813333333333333, "gpqa": null, "omniscience": 26.666666666666668, "omniscienceAccuracy": 0.5385, "omniscienceNonHallucination": 0.41097869266883347, "gdpvalNormalized": 0.46834000000000003, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 872000, "medianOutputTokensPerSecond": 68.8711141116107, "medianTimeToFirstTokenSeconds": 52.2350648785001, "isOpenWeights": false }, { "name": "Qwen3 Coder 30B A3B Instruct", "shortName": "Qwen3 Coder 30B A3B", "slug": "qwen3-coder-30b-a3b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.58537639337592, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.151515151515152, "scicode": null, "hle": 0.0384615384615385, "critpt": 0, "lcr": 0.326666666666667, "gpqa": 0.516161616161616, "omniscience": -50.733333333333334, "omniscienceAccuracy": 0.164, "omniscienceNonHallucination": 0.19696969696969702, "gdpvalNormalized": null, "price1mInputTokens": 0.45, "price1mOutputTokens": 2.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 78.9431092676152, "medianTimeToFirstTokenSeconds": 2.65637602500002, "isOpenWeights": true }, { "name": "K2-V2 (Low)", "shortName": "K2-V2 (low)", "slug": "k2-v2-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 7.31130007284626, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.036144578313253, "critpt": 0, "lcr": 0.203333333333333, "gpqa": 0.541414141414141, "omniscience": -47.28333333333333, "omniscienceAccuracy": 0.16333333333333333, "omniscienceNonHallucination": 0.23964143426294826, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 512000, "isOpenWeights": true }, { "name": "Granite 4.0 H 1B", "shortName": "Granite 4.0 H 1B", "slug": "granite-4-0-h-nano-1b", "isReasoning": false, "deprecated": false, "modelCreatorName": "IBM", "intelligenceIndex": 5.1945517329943, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0500463392029657, "critpt": 0, "lcr": 0.07, "gpqa": 0.262626262626263, "omniscience": -72.26666666666667, "omniscienceAccuracy": 0.052, "omniscienceNonHallucination": 0.18284106891701823, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Reka Flash 3", "shortName": "Reka Flash 3", "slug": "reka-flash-3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Reka AI", "intelligenceIndex": 5.64602230491152, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0444, "critpt": 0, "lcr": 0, "gpqa": 0.529292929292929, "omniscience": -65.13333333333334, "omniscienceAccuracy": 0.13666666666666666, "omniscienceNonHallucination": 0.08725868725868724, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "DeepSeek V4.1 Flash (Max)", "shortName": "DeepSeek V4.1 Flash (max)", "slug": "deepseek-v4-1-flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "DeepSeek", "intelligenceIndex": 39.456167472527, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.26522527009606844, "terminalBench40": 0.267676767676768, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.518518518518518, "hle": 0.392493049119555, "critpt": 0.142857142857143, "lcr": 0.84, "gpqa": null, "omniscience": -5.3, "omniscienceAccuracy": 0.464, "omniscienceNonHallucination": 0.035447761194029814, "gdpvalNormalized": 0.55, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.006, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 209.383766021677, "medianTimeToFirstTokenSeconds": 0.953613005000079, "isOpenWeights": true }, { "name": "GPT-5.6 Luna (Non-reasoning)", "shortName": "GPT-5.6 Luna (non-reasoning)", "slug": "gpt-5-6-luna-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 15.5270002110259, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.010091821810180262, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.389513108614232, "terminalbenchHard": null, "scicode": 0.403935185185185, "hle": 0.072289156626506, "critpt": 0.00285714285714286, "lcr": 0.426666666666667, "gpqa": 0.645454545454546, "omniscience": -24.95, "omniscienceAccuracy": 0.286, "omniscienceNonHallucination": 0.25, "gdpvalNormalized": 0.20308999999999996, "price1mInputTokens": 0.2, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.02, "cacheWritePrice": 0.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 107.447732127516, "medianTimeToFirstTokenSeconds": 0.740743556499993, "isOpenWeights": false }, { "name": "Mistral Medium 3.5", "shortName": "Mistral Medium 3.5", "slug": "mistral-medium-3-5", "isReasoning": true, "deprecated": false, "modelCreatorName": "Mistral", "intelligenceIndex": 14.1889227802691, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5020382394983653, "terminalBench40": 0, "terminalBench21": 0.50561797752809, "terminalbenchHard": 0.333333333333333, "scicode": 0.40162037037037, "hle": 0.137627432808156, "critpt": 0, "lcr": 0.693333333333333, "gpqa": 0.748484848484849, "omniscience": -36.8, "omniscienceAccuracy": 0.24666666666666667, "omniscienceNonHallucination": 0.184070796460177, "gdpvalNormalized": 0.12355000000000001, "price1mInputTokens": 1.5, "price1mOutputTokens": 7.5, "cacheHitPrice": 0.15, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 165.42076427647, "medianTimeToFirstTokenSeconds": 2.295845279, "isOpenWeights": true }, { "name": "K2-V2 (Medium)", "shortName": "K2-V2 (medium)", "slug": "k2-v2-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Institute of Foundation Models", "intelligenceIndex": 9.01595166538807, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0833333333333333, "scicode": null, "hle": 0.0435588507877665, "critpt": 0, "lcr": 0.28, "gpqa": 0.597979797979798, "omniscience": -49, "omniscienceAccuracy": 0.17933333333333334, "omniscienceNonHallucination": 0.18440292445166528, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 512000, "isOpenWeights": true }, { "name": "Claude 2.0", "shortName": "Claude 2.0", "slug": "claude-2", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 5.50606445861584, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": 0.344444444444444, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 100000, "isOpenWeights": false }, { "name": "Gemini 3.7 Flash (Low)", "shortName": "Gemini 3.7 Flash (low)", "slug": "gemini-3-7-flash-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 36.9459114891198, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.797752808988764, "terminalbenchHard": null, "scicode": 0.556712962962963, "hle": 0.351251158480074, "critpt": 0.0571428571428571, "lcr": 0.786666666666667, "gpqa": 0.901010101010101, "omniscience": 22.133333333333333, "omniscienceAccuracy": 0.5356666666666666, "omniscienceNonHallucination": 0.3230437903804738, "gdpvalNormalized": 0.402615, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.075, "cacheWritePrice": 0.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 290.16955157217, "medianTimeToFirstTokenSeconds": 1.18091109499995, "isOpenWeights": false }, { "name": "Pixtral Large", "shortName": "Pixtral Large", "slug": "pixtral-large-2411", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 7.14734063408637, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.028, "critpt": null, "lcr": null, "gpqa": 0.505050505050505, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Qwen3 VL 8B (Reasoning)", "shortName": "Qwen3 VL 8B", "slug": "qwen3-vl-8b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 8.17229749994459, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0384615384615385, "critpt": 0.00285714285714286, "lcr": 0.333333333333333, "gpqa": 0.578787878787879, "omniscience": -52.3, "omniscienceAccuracy": 0.20433333333333334, "omniscienceNonHallucination": 0.08588186007540843, "gdpvalNormalized": null, "price1mInputTokens": 0.18, "price1mOutputTokens": 2.1, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 114.104787985407, "medianTimeToFirstTokenSeconds": 2.38738810649999, "isOpenWeights": true }, { "name": "Phi-3 Mini Instruct 3.8B", "shortName": "Phi-3 Mini", "slug": "phi-3-mini", "isReasoning": false, "deprecated": true, "modelCreatorName": "Microsoft", "intelligenceIndex": 5.82401211090128, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0496, "critpt": null, "lcr": null, "gpqa": 0.319191919191919, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "Mi:dm K 2.5 Pro", "shortName": "Mi:dm K 2.5 Pro", "slug": "mi-dm-k-2-5-pro-dec28", "isReasoning": true, "deprecated": false, "modelCreatorName": "Korea Telecom", "intelligenceIndex": 11.0361387091137, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0227272727272727, "scicode": null, "hle": 0.0806302131603336, "critpt": 0, "lcr": 0.09, "gpqa": 0.701010101010101, "omniscience": -59.18333333333333, "omniscienceAccuracy": 0.17883333333333334, "omniscienceNonHallucination": 0.06149786888573172, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Claude Opus 5.5 (Adaptive Reasoning, Xhigh Effort, Default Fallback)", "shortName": "Claude Opus 5.5 (xhigh with fallback)", "slug": "claude-opus-5-5-xhigh", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 55.9873505840139, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 3.459110175822289, "terminalBench40": 0.595959595959596, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.650462962962963, "hle": 0.575069508804449, "critpt": 0.317142857142857, "lcr": 0.846666666666667, "gpqa": null, "omniscience": 42.65, "omniscienceAccuracy": 0.6538333333333334, "omniscienceNonHallucination": 0.34328358208955223, "gdpvalNormalized": 0.660055, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.2, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 78.5446549540035, "medianTimeToFirstTokenSeconds": 137.3388380525, "isOpenWeights": false }, { "name": "Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)", "shortName": "Claude Sonnet 4.6 (max)", "slug": "claude-sonnet-4-6-adaptive", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 30.0575611117644, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.4858655963942224, "terminalBench40": 0.0303030303030303, "terminalBench21": 0.711610486891386, "terminalbenchHard": 0.53030303030303, "scicode": 0.501157407407407, "hle": 0.33595922150139, "critpt": 0.0314285714285714, "lcr": 0.8, "gpqa": 0.874747474747475, "omniscience": 12.216666666666667, "omniscienceAccuracy": 0.4086666666666667, "omniscienceNonHallucination": 0.5155016910935739, "gdpvalNormalized": 0.35986, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 50.5444743222467, "medianTimeToFirstTokenSeconds": 114.1779126735, "isOpenWeights": false }, { "name": "DeepSeek R1 Distill Qwen 32B", "shortName": "DeepSeek R1 Distill Qwen 32B", "slug": "deepseek-r1-distill-qwen-32b", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 8.37720334584862, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0458, "critpt": null, "lcr": 0.0866666666666667, "gpqa": 0.615151515151515, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "DeepSeek V3.1 Terminus (Non-reasoning)", "shortName": "DeepSeek V3.1 Terminus (non-reasoning)", "slug": "deepseek-v3-1-terminus", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 13.930676595341, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.318181818181818, "scicode": null, "hle": 0.0866543095458758, "critpt": 0, "lcr": 0.453333333333333, "gpqa": 0.750505050505051, "omniscience": -43.46666666666667, "omniscienceAccuracy": 0.23516666666666666, "omniscienceNonHallucination": 0.1242100675528438, "gdpvalNormalized": null, "price1mInputTokens": 0.27, "price1mOutputTokens": 1, "cacheHitPrice": 0.135, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Gemini 2.0 Flash Thinking Experimental (Dec '24)", "shortName": "Gemini 2.0 Flash Thinking exp. (Dec)", "slug": "gemini-2-0-flash-thinking-exp-1219", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 6.55531350618149, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2000000, "isOpenWeights": false }, { "name": "GPT-6 Astra (High)", "shortName": "GPT-6 Astra (high)", "slug": "gpt-6-astra-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 50.9191471636152, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.7252530861048456, "terminalBench40": 0.54040404040404, "terminalBench21": 0.898876404494382, "terminalbenchHard": null, "scicode": 0.554398148148148, "hle": 0.530583873957368, "critpt": 0.288571428571429, "lcr": 0.8, "gpqa": 0.94949494949495, "omniscience": 43.733333333333334, "omniscienceAccuracy": 0.6113333333333333, "omniscienceNonHallucination": 0.5523156089193826, "gdpvalNormalized": 0.49226, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 1, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 45.4420708389101, "medianTimeToFirstTokenSeconds": 58.3932155525, "isOpenWeights": false }, { "name": "GPT-5 nano (High)", "shortName": "GPT-5 nano (high)", "slug": "gpt-5-nano", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 12.992236382483, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.121212121212121, "scicode": null, "hle": 0.0949953660797034, "critpt": 0, "lcr": 0.45, "gpqa": 0.675757575757576, "omniscience": -28.666666666666668, "omniscienceAccuracy": 0.19083333333333333, "omniscienceNonHallucination": 0.4098867147270855, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.005, "cacheWritePrice": null, "contextWindowTokens": 400000, "medianOutputTokensPerSecond": 147.373808161813, "medianTimeToFirstTokenSeconds": 89.6001237040001, "isOpenWeights": false }, { "name": "LFM2.5-2.6B", "shortName": "LFM2.5-2.6B", "slug": "lfm2-5-2-6b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Liquid AI", "intelligenceIndex": 8.38708534765242, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.0449438202247191, "terminalbenchHard": null, "scicode": 0.143518518518519, "hle": 0.06209453197405, "critpt": 0, "lcr": 0.0566666666666667, "gpqa": 0.557575757575758, "omniscience": -10.95, "omniscienceAccuracy": 0.0435, "omniscienceNonHallucination": 0.840041819132253, "gdpvalNormalized": 0, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "Llama Nemotron Super 49B v1.5 (Reasoning)", "shortName": "Llama Nemotron Super 49B v1.5", "slug": "llama-nemotron-super-49b-v1-5-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "NVIDIA", "intelligenceIndex": 9.01435430969758, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.053030303030303, "scicode": null, "hle": 0.0732159406858202, "critpt": 0, "lcr": null, "gpqa": 0.748484848484849, "omniscience": -47.6, "omniscienceAccuracy": 0.17166666666666666, "omniscienceNonHallucination": 0.21810865191146878, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "LFM2.5-8B-A1B", "shortName": "LFM2.5-8B-A1B", "slug": "lfm2-5-8b-a1b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Liquid AI", "intelligenceIndex": 7.21643567343487, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0685820203892493, "critpt": 0, "lcr": 0, "gpqa": 0.513131313131313, "omniscience": -33.2, "omniscienceAccuracy": 0.09333333333333334, "omniscienceNonHallucination": 0.5308823529411765, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Ling 3.0 Flash", "shortName": "Ling 3.0 Flash", "slug": "ling-3-0-flash", "isReasoning": true, "deprecated": false, "modelCreatorName": "InclusionAI", "intelligenceIndex": 20.1332521865481, "intelligenceIndexIsEstimated": false, "terminalBench40": 0, "terminalBench21": 0.554307116104869, "terminalbenchHard": null, "scicode": 0.420138888888889, "hle": 0.236793327154773, "critpt": 0.0171428571428571, "lcr": 0.73, "gpqa": 0.854545454545454, "omniscience": -17.883333333333333, "omniscienceAccuracy": 0.18166666666666667, "omniscienceNonHallucination": 0.559470468431772, "gdpvalNormalized": 0.22032999999999997, "price1mInputTokens": 0.075, "price1mOutputTokens": 0.22, "cacheHitPrice": 0.015, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 325.383530825118, "medianTimeToFirstTokenSeconds": 2.83474757850002, "isOpenWeights": true }, { "name": "GPT-5.5 Instant (May 2026)", "shortName": "GPT-5.5 Instant (May 2026)", "slug": "gpt-5-5-instant-05-26", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 22.6863693000789, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.424242424242424, "scicode": null, "hle": 0.215940685820204, "critpt": 0.0257142857142857, "lcr": 0.663333333333333, "gpqa": 0.846464646464646, "omniscience": 11.15, "omniscienceAccuracy": 0.4625, "omniscienceNonHallucination": 0.3469767441860465, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 30, "cacheHitPrice": 0.5, "cacheWritePrice": null, "contextWindowTokens": 400000, "isOpenWeights": false }, { "name": "Qwen2.5 Coder Instruct 7B", "shortName": "Qwen2.5 Coder 7B", "slug": "qwen2-5-coder-7b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.78716786538995, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0489, "critpt": null, "lcr": null, "gpqa": 0.339393939393939, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "isOpenWeights": true }, { "name": "Qwen3 4B (Non-reasoning)", "shortName": "Qwen3 4B (non-reasoning)", "slug": "qwen3-4b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.61043815916972, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0332, "critpt": null, "lcr": null, "gpqa": 0.397979797979798, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "Nova Premier", "shortName": "Nova Premier", "slug": "nova-premier", "isReasoning": false, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 9.15908119897819, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.0417052826691381, "critpt": 0, "lcr": 0.343333333333333, "gpqa": 0.568686868686869, "omniscience": -35.38333333333333, "omniscienceAccuracy": 0.2125, "omniscienceNonHallucination": 0.2808465608465609, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Mistral Small 3", "shortName": "Mistral Small 3", "slug": "mistral-small-3", "isReasoning": false, "deprecated": true, "modelCreatorName": "Mistral", "intelligenceIndex": 6.67120856746569, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0376, "critpt": null, "lcr": 0, "gpqa": 0.461616161616162, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.08, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32000, "isOpenWeights": true }, { "name": "GPT-6.1 Sol (Low)", "shortName": "GPT-6.1 Sol (low)", "slug": "gpt-6-1-sol-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 42.0835618555848, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.13075190869859377, "terminalBench40": 0.308080808080808, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.532407407407407, "hle": 0.473586654309546, "critpt": 0.248571428571429, "lcr": 0.84, "gpqa": null, "omniscience": 37.63333333333333, "omniscienceAccuracy": 0.5886666666666667, "omniscienceNonHallucination": 0.48379254457050247, "gdpvalNormalized": 0.3985, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.1, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 59.652214222886, "medianTimeToFirstTokenSeconds": 2.61253805799993, "isOpenWeights": false }, { "name": "GLM-5.2 (Non-reasoning)", "shortName": "GLM-5.2 (non-reasoning)", "slug": "glm-5-2-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 22.4323469590326, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.51685393258427, "terminalbenchHard": null, "scicode": null, "hle": 0.097775718257646, "critpt": 0.0314285714285714, "lcr": 0.423333333333333, "gpqa": 0.685858585858586, "omniscience": -6.566666666666666, "omniscienceAccuracy": 0.20283333333333334, "omniscienceNonHallucination": 0.6631821032824587, "gdpvalNormalized": 0.3665, "price1mInputTokens": 1.4, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.26, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 138.270058404211, "medianTimeToFirstTokenSeconds": 1.31171637400001, "isOpenWeights": true }, { "name": "o1-pro", "shortName": "o1-pro", "slug": "o1-pro", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 12.4164514124427, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 150, "price1mOutputTokens": 600, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Jamba 1.6 Mini", "shortName": "Jamba 1.6 Mini", "slug": "jamba-1-6-mini", "isReasoning": false, "deprecated": true, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 5.16209285058134, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0431, "critpt": null, "lcr": null, "gpqa": 0.3, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "GPT-6 Astra (Medium)", "shortName": "GPT-6 Astra (medium)", "slug": "gpt-6-astra-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 49.5704363034369, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.5406493220021167, "terminalBench40": 0.494949494949495, "terminalBench21": 0.895131086142322, "terminalbenchHard": null, "scicode": 0.541666666666667, "hle": 0.527340129749768, "critpt": 0.291428571428571, "lcr": 0.796666666666667, "gpqa": 0.939393939393939, "omniscience": 42.21666666666667, "omniscienceAccuracy": 0.6056666666666667, "omniscienceNonHallucination": 0.5346576500422655, "gdpvalNormalized": 0.48394000000000004, "price1mInputTokens": 10, "price1mOutputTokens": 50, "cacheHitPrice": 1, "cacheWritePrice": 12.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 44.6262457141486, "medianTimeToFirstTokenSeconds": 6.07362504449998, "isOpenWeights": false }, { "name": "GPT-4o (May '24)", "shortName": "GPT-4o (May)", "slug": "gpt-4o-2024-05-13", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 7.33166741584823, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0176, "critpt": null, "lcr": null, "gpqa": 0.526262626262626, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 5, "price1mOutputTokens": 15, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "medianOutputTokensPerSecond": 100.925455740548, "medianTimeToFirstTokenSeconds": 1.05107276299998, "isOpenWeights": false }, { "name": "Claude Opus 4.7 (Adaptive Reasoning, Max Effort)", "shortName": "Claude Opus 4.7 (max)", "slug": "claude-opus-4-7", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 40.6897928205908, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.831460674157303, "terminalbenchHard": 0.515151515151515, "scicode": null, "hle": 0.423076923076923, "critpt": 0.12, "lcr": 0.786666666666667, "gpqa": 0.914141414141414, "omniscience": 27.266666666666666, "omniscienceAccuracy": 0.48883333333333334, "omniscienceNonHallucination": 0.5771111835670035, "gdpvalNormalized": 0.41921, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 46.5518528939768, "medianTimeToFirstTokenSeconds": 25.339952822, "isOpenWeights": false }, { "name": "Claude Opus 5.5 (Adaptive Reasoning, Medium Effort, Default Fallback)", "shortName": "Claude Opus 5.5 (medium with fallback)", "slug": "claude-opus-5-5-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 51.2434931792768, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.3360093438976588, "terminalBench40": 0.525252525252525, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.592592592592593, "hle": 0.547265987025023, "critpt": 0.277142857142857, "lcr": 0.843333333333333, "gpqa": null, "omniscience": 40.31666666666667, "omniscienceAccuracy": 0.6455, "omniscienceNonHallucination": 0.31640808650681707, "gdpvalNormalized": 0.5381, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.2, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 71.9384212150227, "medianTimeToFirstTokenSeconds": 22.1838609384999, "isOpenWeights": false }, { "name": "GPT-5 (ChatGPT)", "shortName": "GPT-5 (ChatGPT)", "slug": "gpt-5-chatgpt", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 10.4439365979985, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.128787878787879, "scicode": null, "hle": 0.0662650602409639, "critpt": null, "lcr": 0.65, "gpqa": 0.685858585858586, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": false }, { "name": "Gemini 3.7 Flash (Medium)", "shortName": "Gemini 3.7 Flash (medium)", "slug": "gemini-3-7-flash-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 39.617751842555, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.782771535580524, "terminalbenchHard": null, "scicode": 0.59837962962963, "hle": 0.389712696941613, "critpt": 0.0942857142857143, "lcr": 0.83, "gpqa": 0.921212121212121, "omniscience": 23.7, "omniscienceAccuracy": 0.54, "omniscienceNonHallucination": 0.341304347826087, "gdpvalNormalized": 0.41978999999999994, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.075, "cacheWritePrice": 0.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 283.343712302427, "medianTimeToFirstTokenSeconds": 5.62443699899999, "isOpenWeights": false }, { "name": "Ring-2.6-1T", "shortName": "Ring-2.6-1T", "slug": "ring-2-6-1t", "isReasoning": true, "deprecated": false, "modelCreatorName": "InclusionAI", "intelligenceIndex": 16.6172363768618, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.2871573189066596, "terminalBench40": 0.00505050505050505, "terminalBench21": 0.430711610486891, "terminalbenchHard": 0.287878787878788, "scicode": 0.450231481481481, "hle": 0.215940685820204, "critpt": 0.0371428571428571, "lcr": 0.7, "gpqa": 0.856565656565657, "omniscience": -37.7, "omniscienceAccuracy": 0.257, "omniscienceNonHallucination": 0.14670255720053837, "gdpvalNormalized": 0.11680000000000001, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 117.603812865472, "medianTimeToFirstTokenSeconds": 4.13821242200004, "isOpenWeights": true }, { "name": "Solar Pro 2 (Reasoning)", "shortName": "Solar Pro 2", "slug": "solar-pro-2-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Upstage", "intelligenceIndex": 7.49170414062443, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0303030303030303, "scicode": null, "hle": 0.0736793327154773, "critpt": 0, "lcr": 0, "gpqa": 0.686868686868687, "omniscience": -55.88333333333333, "omniscienceAccuracy": 0.1995, "omniscienceNonHallucination": 0.052675411201332456, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 65536, "isOpenWeights": false }, { "name": "Muse Spark", "shortName": "Muse Spark", "slug": "muse-spark", "isReasoning": true, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 31.2991113054345, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.621722846441948, "terminalbenchHard": 0.454545454545455, "scicode": null, "hle": 0.406858202038925, "critpt": 0.11330612244898, "lcr": 0.78, "gpqa": 0.883838383838384, "omniscience": 7.166666666666667, "omniscienceAccuracy": 0.496, "omniscienceNonHallucination": 0.15806878306878303, "gdpvalNormalized": 0.24289, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "isOpenWeights": false }, { "name": "Grok 3 Reasoning Beta", "shortName": "Grok 3 Reasoning Beta", "slug": "grok-3-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 10.3553917819163, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "OpenChat 3.5 (1210)", "shortName": "OpenChat 3.5", "slug": "openchat-35", "isReasoning": false, "deprecated": true, "modelCreatorName": "OpenChat", "intelligenceIndex": 5.28995755703256, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0479, "critpt": null, "lcr": null, "gpqa": 0.23030303030303, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 8192, "isOpenWeights": true }, { "name": "Qwen3 14B (Non-reasoning)", "shortName": "Qwen3 14B (non-reasoning)", "slug": "qwen3-14b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.7030362726921, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.053030303030303, "scicode": null, "hle": 0.0409, "critpt": 0, "lcr": 0, "gpqa": 0.46969696969697, "omniscience": -66.66666666666667, "omniscienceAccuracy": 0.13333333333333333, "omniscienceNonHallucination": 0.07692307692307687, "gdpvalNormalized": null, "price1mInputTokens": 0.35, "price1mOutputTokens": 1.4, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 63.3663164065549, "medianTimeToFirstTokenSeconds": 2.74724775350001, "isOpenWeights": true }, { "name": "Phi-4", "shortName": "Phi-4", "slug": "phi-4", "isReasoning": false, "deprecated": false, "modelCreatorName": "Microsoft", "intelligenceIndex": 5.92257028113751, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0378787878787879, "scicode": null, "hle": 0.0376, "critpt": 0, "lcr": 0, "gpqa": 0.574747474747475, "omniscience": -55.7, "omniscienceAccuracy": 0.14066666666666666, "omniscienceNonHallucination": 0.18813033359193176, "gdpvalNormalized": null, "price1mInputTokens": 0.125, "price1mOutputTokens": 0.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 16000, "medianOutputTokensPerSecond": 39.5900836128083, "medianTimeToFirstTokenSeconds": 2.63674649250001, "isOpenWeights": true }, { "name": "Qwen3.5 35B A3B (Non-reasoning)", "shortName": "Qwen3.5 35B A3B (non-reasoning)", "slug": "qwen3-5-35b-a3b-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 15.1159818387307, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.408239700374532, "terminalbenchHard": 0.106060606060606, "scicode": null, "hle": 0.134383688600556, "critpt": 0.00571428571428571, "lcr": 0.63, "gpqa": 0.819191919191919, "omniscience": -63.016666666666666, "omniscienceAccuracy": 0.15716666666666668, "omniscienceNonHallucination": 0.06584931777733838, "gdpvalNormalized": 0.04732999999999998, "price1mInputTokens": 0.25, "price1mOutputTokens": 2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262144, "medianOutputTokensPerSecond": 155.836375408294, "medianTimeToFirstTokenSeconds": 2.09137940200001, "isOpenWeights": true }, { "name": "Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)", "shortName": "Claude Sonnet 5 (medium)", "slug": "claude-sonnet-5-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 28.0531468379456, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.9991061406688175, "terminalBench40": 0.0202020202020202, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.516203703703704, "hle": 0.299814643188137, "critpt": 0.0857142857142857, "lcr": 0.736666666666667, "gpqa": null, "omniscience": -6.866666666666666, "omniscienceAccuracy": 0.371, "omniscienceNonHallucination": 0.30100688924218333, "gdpvalNormalized": 0.32238, "price1mInputTokens": 2, "price1mOutputTokens": 10, "cacheHitPrice": 0.2, "cacheWritePrice": 2.5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 58.4222927087695, "medianTimeToFirstTokenSeconds": 1.607736681, "isOpenWeights": false }, { "name": "gpt-oss-120b (High)", "shortName": "gpt-oss-120b (high)", "slug": "gpt-oss-120b", "isReasoning": true, "deprecated": false, "modelCreatorName": "OpenAI", "intelligenceIndex": 11.6028431512592, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.10742452290394947, "terminalBench40": 0, "terminalBench21": 0.262172284644195, "terminalbenchHard": 0.234848484848485, "scicode": 0.340277777777778, "hle": 0.196014828544949, "critpt": 0.0114285714285714, "lcr": 0.52, "gpqa": 0.781818181818182, "omniscience": -49.25, "omniscienceAccuracy": 0.21783333333333332, "omniscienceNonHallucination": 0.09183890901342429, "gdpvalNormalized": 0.04775, "price1mInputTokens": 0.15, "price1mOutputTokens": 0.595, "cacheHitPrice": 0.125, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 173.566990204574, "medianTimeToFirstTokenSeconds": 0.844251966999991, "isOpenWeights": true }, { "name": "GLM-5 (Non-reasoning)", "shortName": "GLM-5 (non-reasoning)", "slug": "glm-5-non-reasoning", "isReasoning": false, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 21.7826479737062, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.393939393939394, "scicode": null, "hle": 0.0764596848934198, "critpt": 0, "lcr": 0.436666666666667, "gpqa": 0.665656565656566, "omniscience": -11.8, "omniscienceAccuracy": 0.2305, "omniscienceNonHallucination": 0.5471085120207928, "gdpvalNormalized": null, "price1mInputTokens": 1, "price1mOutputTokens": 3.2, "cacheHitPrice": 0.2, "cacheWritePrice": null, "contextWindowTokens": 200000, "medianOutputTokensPerSecond": 67.3771895328171, "medianTimeToFirstTokenSeconds": 1.64880107900006, "isOpenWeights": true }, { "name": "NVIDIA Nemotron Nano 9B V2 (Reasoning)", "shortName": "NVIDIA Nemotron Nano 9B V2", "slug": "nvidia-nemotron-nano-9b-v2-reasoning", "isReasoning": true, "deprecated": false, "modelCreatorName": "NVIDIA", "intelligenceIndex": 7.42882284148722, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0486561631139944, "critpt": 0, "lcr": 0.226666666666667, "gpqa": 0.56969696969697, "omniscience": -41.55, "omniscienceAccuracy": 0.118, "omniscienceNonHallucination": 0.39512471655328796, "gdpvalNormalized": null, "price1mInputTokens": 0.04, "price1mOutputTokens": 0.16, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 131072, "medianOutputTokensPerSecond": 103.858641538459, "medianTimeToFirstTokenSeconds": 5.11697496300004, "isOpenWeights": true }, { "name": "Claude Sonnet 4.6 (Non-reasoning, Low Effort)", "shortName": "Claude Sonnet 4.6 (non-reasoning, low)", "slug": "claude-sonnet-4-6-non-reasoning-low-effort", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 23.3035011715443, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.424242424242424, "scicode": null, "hle": 0.112140871177016, "critpt": 0.00857142857142857, "lcr": 0.693333333333333, "gpqa": 0.796969696969697, "omniscience": -2.1, "omniscienceAccuracy": 0.36516666666666664, "omniscienceNonHallucination": 0.3917038592806511, "gdpvalNormalized": null, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": 3.75, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 43.0389870539269, "medianTimeToFirstTokenSeconds": 1.16052743449995, "isOpenWeights": false }, { "name": "Claude 4.1 Opus (Non-reasoning)", "shortName": "Claude 4.1 Opus (non-reasoning)", "slug": "claude-4-1-opus", "isReasoning": false, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 18.5669784317462, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 15, "price1mOutputTokens": 75, "cacheHitPrice": 1.5, "cacheWritePrice": 18.75, "contextWindowTokens": 200000, "isOpenWeights": false }, { "name": "Claude Opus 5.5 (Adaptive Reasoning, High Effort, Default Fallback)", "shortName": "Claude Opus 5.5 (high with fallback)", "slug": "claude-opus-5-5-high", "isReasoning": true, "deprecated": false, "modelCreatorName": "Anthropic", "intelligenceIndex": 53.5831959822067, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.822504718771704, "terminalBench40": 0.565656565656566, "terminalBench21": null, "terminalbenchHard": null, "scicode": 0.604166666666667, "hle": 0.555607043558851, "critpt": 0.308571428571429, "lcr": 0.826666666666667, "gpqa": null, "omniscience": 40.63333333333333, "omniscienceAccuracy": 0.6458333333333334, "omniscienceNonHallucination": 0.32376470588235295, "gdpvalNormalized": 0.5958600000000001, "price1mInputTokens": 4, "price1mOutputTokens": 20, "cacheHitPrice": 0.2, "cacheWritePrice": 5, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 74.3972614063985, "medianTimeToFirstTokenSeconds": 37.3713814654999, "isOpenWeights": false }, { "name": "Qwen3 30B A3B (Non-reasoning)", "shortName": "Qwen3 30B (non-reasoning)", "slug": "qwen3-30b-a3b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 6.62259542880934, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0681818181818182, "scicode": null, "hle": 0.046, "critpt": 0, "lcr": 0, "gpqa": 0.515151515151515, "omniscience": -65.98333333333333, "omniscienceAccuracy": 0.11966666666666667, "omniscienceNonHallucination": 0.11453994698977665, "gdpvalNormalized": null, "price1mInputTokens": 0.2, "price1mOutputTokens": 0.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "medianOutputTokensPerSecond": 105.481307846576, "medianTimeToFirstTokenSeconds": 2.25089720849996, "isOpenWeights": true }, { "name": "Qwen3.6 Plus", "shortName": "Qwen3.6 Plus", "slug": "qwen3-6-plus", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 27.0099232636747, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.614232209737828, "terminalbenchHard": 0.439393939393939, "scicode": null, "hle": 0.278498609823911, "critpt": 0.0285714285714286, "lcr": 0.783333333333333, "gpqa": 0.881818181818182, "omniscience": 0.8833333333333333, "omniscienceAccuracy": 0.2638333333333333, "omniscienceNonHallucination": 0.6536110482227757, "gdpvalNormalized": 0.23792000000000002, "price1mInputTokens": 0.5, "price1mOutputTokens": 3, "cacheHitPrice": 0.05, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 56.0486977743491, "medianTimeToFirstTokenSeconds": 2.12381014400001, "isOpenWeights": false }, { "name": "Nova 2.0 Pro Preview (Low)", "shortName": "Nova 2.0 Pro Preview (low)", "slug": "nova-2-0-pro-reasoning-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 12.7754507441167, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.194756554307116, "terminalbenchHard": 0.174242424242424, "scicode": null, "hle": 0.0518999073215941, "critpt": 0, "lcr": 0.646666666666667, "gpqa": 0.750505050505051, "omniscience": -46.53333333333333, "omniscienceAccuracy": 0.22, "omniscienceNonHallucination": 0.12136752136752138, "gdpvalNormalized": 0, "price1mInputTokens": 1.25, "price1mOutputTokens": 10, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 123.967836929987, "medianTimeToFirstTokenSeconds": 11.2835517094999, "isOpenWeights": false }, { "name": "Gemini 2.5 Flash-Lite (Reasoning)", "shortName": "Gemini 2.5 Flash-Lite", "slug": "gemini-2-5-flash-lite-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 8.54563691769003, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0454545454545455, "scicode": null, "hle": 0.0681186283595922, "critpt": 0, "lcr": 0.556666666666667, "gpqa": 0.625252525252525, "omniscience": -45.61666666666667, "omniscienceAccuracy": 0.179, "omniscienceNonHallucination": 0.2263499796995534, "gdpvalNormalized": null, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.4, "cacheHitPrice": 0.01, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 345.411227162436, "medianTimeToFirstTokenSeconds": 17.8030958894999, "isOpenWeights": false }, { "name": "Qwen3 VL 4B Instruct", "shortName": "Qwen3 VL 4B", "slug": "qwen3-vl-4b-instruct", "isReasoning": false, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 5.65532227476457, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.036144578313253, "critpt": 0, "lcr": 0.14, "gpqa": 0.370707070707071, "omniscience": -75.96666666666667, "omniscienceAccuracy": 0.10866666666666666, "omniscienceNonHallucination": 0.025804038893044168, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Qwen3 235B A22B 2507 (Reasoning)", "shortName": "Qwen3 235B A22B 2507", "slug": "qwen3-235b-a22b-instruct-2507-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 12.7108511527974, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.08055542892015415, "terminalBench40": 0, "terminalBench21": 0.119850187265918, "terminalbenchHard": 0.136363636363636, "scicode": 0.414351851851852, "hle": 0.158943466172382, "critpt": 0, "lcr": 0.72, "gpqa": 0.78989898989899, "omniscience": -46.516666666666666, "omniscienceAccuracy": 0.22816666666666666, "omniscienceNonHallucination": 0.10170589505506367, "gdpvalNormalized": 0, "price1mInputTokens": 0.23, "price1mOutputTokens": 2.3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 65.2306029671905, "medianTimeToFirstTokenSeconds": 2.83119383900004, "isOpenWeights": true }, { "name": "Gemini 3.8 Flash (Low)", "shortName": "Gemini 3.8 Flash (low)", "slug": "gemini-3-8-flash-low", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 33.4548106699132, "intelligenceIndexIsEstimated": false, "terminalBench40": 0.101010101010101, "terminalBench21": 0.831460674157303, "terminalbenchHard": null, "scicode": 0.549768518518518, "hle": 0.370713623725672, "critpt": 0.04, "lcr": 0.806666666666667, "gpqa": 0.92020202020202, "omniscience": 21.35, "omniscienceAccuracy": 0.5221666666666667, "omniscienceNonHallucination": 0.35402860132542724, "gdpvalNormalized": 0.39718, "price1mInputTokens": 0.75, "price1mOutputTokens": 3.75, "cacheHitPrice": 0.075, "cacheWritePrice": 0.75, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Llama 3.1 Tulu3 405B", "shortName": "Tulu3 405B", "slug": "tulu3-405b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 7.19889208713507, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0328, "critpt": null, "lcr": null, "gpqa": 0.516161616161616, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "MiniMax-M2", "shortName": "MiniMax-M2", "slug": "minimax-m2", "isReasoning": true, "deprecated": true, "modelCreatorName": "MiniMax", "intelligenceIndex": 18.6256708551149, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.257575757575758, "scicode": null, "hle": 0.136700648748842, "critpt": 0.00857142857142857, "lcr": 0.643333333333333, "gpqa": 0.776767676767677, "omniscience": -46.55, "omniscienceAccuracy": 0.23233333333333334, "omniscienceNonHallucination": 0.0909683022145028, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 204800, "medianOutputTokensPerSecond": 86.6408528146247, "medianTimeToFirstTokenSeconds": 1.52864984199982, "isOpenWeights": true }, { "name": "Jamba Reasoning 3B", "shortName": "Jamba Reasoning 3B", "slug": "jamba-reasoning-3b", "isReasoning": true, "deprecated": false, "modelCreatorName": "AI21 Labs", "intelligenceIndex": 5.66829948484426, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.00757575757575758, "scicode": null, "hle": 0.0375347544022243, "critpt": 0, "lcr": 0.0633333333333333, "gpqa": 0.333333333333333, "omniscience": -57.71666666666667, "omniscienceAccuracy": 0.067, "omniscienceNonHallucination": 0.3095748481600572, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 262000, "isOpenWeights": true }, { "name": "GLM-5.2 (Max)", "shortName": "GLM-5.2 (max)", "slug": "glm-5-2", "isReasoning": true, "deprecated": true, "modelCreatorName": "Z AI", "intelligenceIndex": 33.7054572377545, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 1.4726179494034382, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.779026217228464, "terminalbenchHard": 0.507575757575758, "scicode": 0.511574074074074, "hle": 0.411492122335496, "critpt": 0.208571428571429, "lcr": 0.783333333333333, "gpqa": 0.894949494949495, "omniscience": 4.433333333333334, "omniscienceAccuracy": 0.24333333333333335, "omniscienceNonHallucination": 0.7370044052863436, "gdpvalNormalized": 0.42876, "price1mInputTokens": 1.4, "price1mOutputTokens": 4.4, "cacheHitPrice": 0.26, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 84.219982249121, "medianTimeToFirstTokenSeconds": 3.09938497300004, "isOpenWeights": true }, { "name": "Kimi K3 (Max)", "shortName": "Kimi K3 (max)", "slug": "kimi-k3", "isReasoning": true, "deprecated": false, "modelCreatorName": "Kimi", "intelligenceIndex": 43.5938229518782, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.0001323004425493, "terminalBench40": 0.126262626262626, "terminalBench21": 0.850187265917603, "terminalbenchHard": null, "scicode": 0.594907407407407, "hle": 0.468952734012975, "critpt": 0.234285714285714, "lcr": 0.886666666666667, "gpqa": 0.935353535353535, "omniscience": 19.7, "omniscienceAccuracy": 0.47583333333333333, "omniscienceNonHallucination": 0.46804451510333867, "gdpvalNormalized": 0.519995, "price1mInputTokens": 3, "price1mOutputTokens": 15, "cacheHitPrice": 0.3, "cacheWritePrice": null, "contextWindowTokens": 1048576, "medianOutputTokensPerSecond": 33.7371270862654, "medianTimeToFirstTokenSeconds": 4.53207441900004, "isOpenWeights": true }, { "name": "OLMo 2 7B", "shortName": "OLMo 2 7B", "slug": "olmo-2-7b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Allen Institute for AI", "intelligenceIndex": 5.57740289284295, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0537534754402224, "critpt": null, "lcr": 0, "gpqa": 0.287878787878788, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true }, { "name": "Qwen3 VL 4B (Reasoning)", "shortName": "Qwen3 VL 4B", "slug": "qwen3-vl-4b-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "Alibaba", "intelligenceIndex": 7.04880477911409, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0151515151515152, "scicode": null, "hle": 0.0458758109360519, "critpt": 0, "lcr": 0.213333333333333, "gpqa": 0.493939393939394, "omniscience": -68.38333333333334, "omniscienceAccuracy": 0.12216666666666667, "omniscienceNonHallucination": 0.0818302639073476, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": true }, { "name": "Quasar 438B (Max, Based on GLM-5.2)", "shortName": "Quasar 438B (max)", "slug": "quasar-438b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Multiverse Computing", "intelligenceIndex": 26.7372536607816, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.024611832657714, "terminalBench40": 0.0101010101010101, "terminalBench21": 0.692883895131086, "terminalbenchHard": null, "scicode": 0.481481481481481, "hle": 0.186746987951807, "critpt": 0.0942857142857143, "lcr": 0.763333333333333, "gpqa": 0.732323232323232, "omniscience": -2.6, "omniscienceAccuracy": 0.15516666666666667, "omniscienceNonHallucination": 0.7855592819096469, "gdpvalNormalized": 0.33079, "price1mInputTokens": 0.6, "price1mOutputTokens": 1.8, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 142.874874359554, "medianTimeToFirstTokenSeconds": 1.15476514749997, "isOpenWeights": false }, { "name": "KAT-Coder-Pro V1", "shortName": "KAT-Coder-Pro V1", "slug": "kat-coder-pro-v1", "isReasoning": false, "deprecated": true, "modelCreatorName": "KwaiKAT", "intelligenceIndex": 18.5862731794299, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.0909090909090909, "scicode": null, "hle": 0.336422613531047, "critpt": 0, "lcr": 0.386666666666667, "gpqa": 0.763636363636364, "omniscience": -37.733333333333334, "omniscienceAccuracy": 0.177, "omniscienceNonHallucination": 0.3264479546375051, "gdpvalNormalized": 0.10916500000000003, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "DeepSeek V3.1 (Non-reasoning)", "shortName": "DeepSeek V3.1 (non-reasoning)", "slug": "deepseek-v3-1", "isReasoning": false, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 13.707850604044, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.242424242424242, "scicode": null, "hle": 0.0667284522706209, "critpt": 0, "lcr": 0.47, "gpqa": 0.735353535353535, "omniscience": -42.75, "omniscienceAccuracy": 0.23133333333333334, "omniscienceNonHallucination": 0.1428881179531657, "gdpvalNormalized": null, "price1mInputTokens": 0.56, "price1mOutputTokens": 1.68, "cacheHitPrice": 0.56, "cacheWritePrice": null, "contextWindowTokens": 128000, "isOpenWeights": true }, { "name": "LFM2 24B A2B", "shortName": "LFM2 24B A2B", "slug": "lfm2-24b-a2b", "isReasoning": false, "deprecated": false, "modelCreatorName": "Liquid AI", "intelligenceIndex": 5.94753271655266, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0, "scicode": null, "hle": 0.0421686746987952, "critpt": 0, "lcr": 0, "gpqa": 0.473737373737374, "omniscience": -58.1, "omniscienceAccuracy": 0.06466666666666666, "omniscienceNonHallucination": 0.3096935138987883, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 32768, "isOpenWeights": true }, { "name": "Nova 2.0 Lite (Medium)", "shortName": "Nova 2.0 Lite (medium)", "slug": "nova-2-0-lite-reasoning-medium", "isReasoning": true, "deprecated": false, "modelCreatorName": "Amazon", "intelligenceIndex": 12.4823794919822, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.174242424242424, "scicode": null, "hle": 0.0898980537534754, "critpt": 0, "lcr": 0.6, "gpqa": 0.767676767676768, "omniscience": -56.96666666666667, "omniscienceAccuracy": 0.18166666666666667, "omniscienceNonHallucination": 0.08187372708757634, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 2.5, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 179.330997256501, "medianTimeToFirstTokenSeconds": 14.2811994705, "isOpenWeights": false }, { "name": "KAT Coder Pro V2", "shortName": "KAT-Coder-Pro V2", "slug": "kat-coder-pro-v2", "isReasoning": false, "deprecated": false, "modelCreatorName": "KwaiKAT", "intelligenceIndex": 21.7045082420985, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.700374531835206, "terminalbenchHard": 0.492424242424242, "scicode": null, "hle": 0.161260426320667, "critpt": 0, "lcr": 0.73, "gpqa": 0.854545454545454, "omniscience": -22.466666666666665, "omniscienceAccuracy": 0.21483333333333332, "omniscienceNonHallucination": 0.44024623222245807, "gdpvalNormalized": 0.10990499999999997, "price1mInputTokens": 0.3, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.06, "cacheWritePrice": null, "contextWindowTokens": 256000, "isOpenWeights": false }, { "name": "Qwen3 Coder Next", "shortName": "Qwen3 Coder Next", "slug": "qwen3-coder-next", "isReasoning": false, "deprecated": false, "modelCreatorName": "Alibaba", "intelligenceIndex": 9.23682203219174, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.5518041274940361, "terminalBench40": 0, "terminalBench21": 0.382022471910112, "terminalbenchHard": 0.181818181818182, "scicode": 0.362268518518519, "hle": 0.101482854494903, "critpt": 0, "lcr": 0.47, "gpqa": 0.737373737373737, "omniscience": -62.4, "omniscienceAccuracy": 0.1615, "omniscienceNonHallucination": 0.06320810971973767, "gdpvalNormalized": 0.0004950000000000046, "price1mInputTokens": 0.35, "price1mOutputTokens": 1.2, "cacheHitPrice": 0.35, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 74.7058468660414, "medianTimeToFirstTokenSeconds": 1.62608414849994, "isOpenWeights": true }, { "name": "GPT-5.1 Codex mini (High)", "shortName": "GPT-5.1 Codex mini (high)", "slug": "gpt-5-1-codex-mini", "isReasoning": true, "deprecated": true, "modelCreatorName": "OpenAI", "intelligenceIndex": 20.3781988302265, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.333333333333333, "scicode": null, "hle": 0.184893419833179, "critpt": 0, "lcr": 0.666666666666667, "gpqa": 0.813131313131313, "omniscience": -16.416666666666668, "omniscienceAccuracy": 0.232, "omniscienceNonHallucination": 0.48415798611111116, "gdpvalNormalized": null, "price1mInputTokens": 0.25, "price1mOutputTokens": 2, "cacheHitPrice": 0.025, "cacheWritePrice": null, "contextWindowTokens": 400000, "isOpenWeights": false }, { "name": "Gemini 2.0 Flash (experimental)", "shortName": "Gemini 2.0 Flash (exp)", "slug": "gemini-2-0-flash-experimental", "isReasoning": false, "deprecated": true, "modelCreatorName": "Google", "intelligenceIndex": 8.21731795326663, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0407, "critpt": null, "lcr": null, "gpqa": 0.636363636363636, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0, "price1mOutputTokens": 0, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 1000000, "isOpenWeights": false }, { "name": "Llama 65B", "shortName": "Llama 65B", "slug": "llama-65b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 5.03498213171439, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": null, "critpt": null, "lcr": null, "gpqa": null, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": null, "price1mOutputTokens": null, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 2048, "isOpenWeights": true }, { "name": "Grok 4.3 (Low)", "shortName": "Grok 4.3 (low)", "slug": "grok-4-3-low", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 24.2958998564863, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.265151515151515, "scicode": null, "hle": 0.183503243744208, "critpt": 0.00571428571428571, "lcr": 0.74, "gpqa": 0.843434343434343, "omniscience": 13.933333333333334, "omniscienceAccuracy": 0.2668333333333333, "omniscienceNonHallucination": 0.8260968401909525, "gdpvalNormalized": null, "price1mInputTokens": 1.25, "price1mOutputTokens": 2.5, "cacheHitPrice": 0.2, "cacheWritePrice": 1.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 119.941046003767, "medianTimeToFirstTokenSeconds": 6.14171931299995, "isOpenWeights": false }, { "name": "DeepSeek V4 Flash 0731 (Max)", "shortName": "DeepSeek V4 Flash 0731 (max)", "slug": "deepseek-v4-flash", "isReasoning": true, "deprecated": true, "modelCreatorName": "DeepSeek", "intelligenceIndex": 34.3305129347816, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 0.21964484501171036, "terminalBench40": 0.121212121212121, "terminalBench21": 0.786516853932584, "terminalbenchHard": null, "scicode": 0.503472222222222, "hle": 0.385542168674699, "critpt": 0.165714285714286, "lcr": 0.796666666666667, "gpqa": 0.908080808080808, "omniscience": -14.283333333333333, "omniscienceAccuracy": 0.4038333333333333, "omniscienceNonHallucination": 0.0830304724629578, "gdpvalNormalized": 0.46328499999999995, "price1mInputTokens": 0.44, "price1mOutputTokens": 1.32, "cacheHitPrice": 0.014, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 216.334229982997, "medianTimeToFirstTokenSeconds": 0.974565590999986, "isOpenWeights": true }, { "name": "Gemma 4 12B (Reasoning)", "shortName": "Gemma 4 12B", "slug": "gemma-4-12b", "isReasoning": true, "deprecated": false, "modelCreatorName": "Google", "intelligenceIndex": 14.1847516647736, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": 0.273408239700375, "terminalbenchHard": 0.181818181818182, "scicode": null, "hle": 0.156626506024096, "critpt": 0, "lcr": 0.636666666666667, "gpqa": 0.752525252525252, "omniscience": -52.71666666666667, "omniscienceAccuracy": 0.15616666666666668, "omniscienceNonHallucination": 0.19020343669761008, "gdpvalNormalized": 0, "price1mInputTokens": 0.1, "price1mOutputTokens": 0.3, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 256000, "medianOutputTokensPerSecond": 111.280423305725, "medianTimeToFirstTokenSeconds": 2.42692915499998, "isOpenWeights": true }, { "name": "Claude Opus 5 (Adaptive Reasoning, Medium Effort)", "shortName": "Claude Opus 5 (medium)", "slug": "claude-opus-5-medium", "isReasoning": true, "deprecated": true, "modelCreatorName": "Anthropic", "intelligenceIndex": 44.8253265420613, "intelligenceIndexIsEstimated": false, "intelligenceIndexCostPerTask": 2.1894793086538695, "terminalBench40": 0.343434343434343, "terminalBench21": 0.861423220973783, "terminalbenchHard": null, "scicode": 0.515046296296296, "hle": 0.512974976830398, "critpt": 0.268571428571429, "lcr": 0.82, "gpqa": 0.919191919191919, "omniscience": 31.016666666666666, "omniscienceAccuracy": 0.5706666666666667, "omniscienceNonHallucination": 0.3932453416149069, "gdpvalNormalized": 0.48796500000000004, "price1mInputTokens": 5, "price1mOutputTokens": 25, "cacheHitPrice": 0.5, "cacheWritePrice": 6.25, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 52.089339821944, "medianTimeToFirstTokenSeconds": 5.26676698099982, "isOpenWeights": false }, { "name": "Grok 3 mini Reasoning (High)", "shortName": "Grok 3 mini Reasoning (high)", "slug": "grok-3-mini-reasoning", "isReasoning": true, "deprecated": true, "modelCreatorName": "SpaceXAI", "intelligenceIndex": 14.6212806219199, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": 0.174242424242424, "scicode": null, "hle": 0.1101, "critpt": 0.00571428571428571, "lcr": 0.553333333333333, "gpqa": 0.790909090909091, "omniscience": -6.8, "omniscienceAccuracy": 0.15116666666666667, "omniscienceNonHallucination": 0.7418024739838995, "gdpvalNormalized": null, "price1mInputTokens": 0.3, "price1mOutputTokens": 0.5, "cacheHitPrice": 0.075, "cacheWritePrice": null, "contextWindowTokens": 1000000, "medianOutputTokensPerSecond": 63.8357597993419, "medianTimeToFirstTokenSeconds": 2.41820521800005, "isOpenWeights": false }, { "name": "Llama 2 Chat 7B", "shortName": "Llama 2 Chat 7B", "slug": "llama-2-chat-7b", "isReasoning": false, "deprecated": true, "modelCreatorName": "Meta", "intelligenceIndex": 5.71179615992055, "intelligenceIndexIsEstimated": true, "terminalBench40": null, "terminalBench21": null, "terminalbenchHard": null, "scicode": null, "hle": 0.0533, "critpt": null, "lcr": null, "gpqa": 0.227272727272727, "omniscience": null, "gdpvalNormalized": null, "price1mInputTokens": 0.05, "price1mOutputTokens": 0.25, "cacheHitPrice": null, "cacheWritePrice": null, "contextWindowTokens": 4096, "isOpenWeights": true } ], "note": "Coding: all configs verbatim. Intelligence: slimmed rows carrying an index score or terminalBench40 (also the source of aa_terminal_bench_4); full payload at _build/aa-update-round5/intel_raw_all.json." } }