{ "as_of": "2026-10-01", "source": "https://artificialanalysis.ai/agents/coding-agents", "benchmark_version": "1.5", "html_sha256": "600ebc51f020ae691ffec0d9604ed3fa330168e0a87d0aab2ba90a1fae24dd1d", "rows": [ { "id": "cd60470c35fd626a116af80fd5915dfa", "isDefault": false, "isHighlighted": false, "agentName": "Muse Code 1.0.2 RC", "provider": "meta", "hostModelSlug": "meta_joyful-jello138", "display": { "agent": "Muse Code", "model": "Muse Spark 1.3 (xhigh)", "creator": { "agent": "Meta", "model": "Meta" } }, "displayLabel": "Muse Code - Muse Spark 1.3 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.731563421828909, "inputTokens": 15210161.250737464, "outputTokens": 78929.04719764012 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.545698924731183, "inputTokens": 3764504.801075269, "outputTokens": 21519.38440860215 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 35, "reviewedAttemptCount": 35, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 35 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.171717171717172, "inputTokens": 40760107.72474747, "outputTokens": 493004.18434343435 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.482993172759088, "mean": { "reward": 0.482993172759088, "costUsd": 3.467754041474145, "agentWallTimeSec": 1098.202911991198, "steps": 187.48954895489547, "inputTokens": 16091465.103960397, "outputTokens": 145629.2480748075, "cacheTokens": 15709288.80143016, "cacheHitRate": 0.9661832360411203, "totalTokens": 16237094.352035202 }, "sums": { "costUsd": 1050.7294745666659 }, "percentiles": { "costUsd": { "p05": 0.6543826890539068, "p25": 1.1450520823432337, "p50": 1.6800104498349842, "p75": 3.3331835402640264, "p95": 11.812090951716167 }, "totalTokens": { "p05": 2805937.5105610583, "p25": 5191748.90649065, "p50": 8172270.941694162, "p75": 15773437.224972516, "p95": 58864615.07931783 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } }, "swe-atlas-qna": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } }, "terminal-bench-v4": { "min": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" }, "max": { "version": "1.0.2-R2040.1", "dateReleased": "2026-09-02" } } } }, { "id": "f927d9c41b5c3021996eb14c3dd1341e", "isDefault": true, "isHighlighted": true, "agentName": "Grok Build", "provider": "xai", "hostModelSlug": "xai_grok-bop-o86-xhigh", "display": { "agent": "Grok Build", "model": "Grok 4.7 (xhigh)", "creator": { "agent": "xAI", "model": "xAI" } }, "displayLabel": "Grok Build - Grok 4.7 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 335, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.725663716814159, "inputTokens": 14656743.734513273, "outputTokens": 152948.2005899705 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.629032258064516, "inputTokens": 7966825.954301075, "outputTokens": 57168.37634408602 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 66, "reviewedAttemptCount": 66, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 66 } ], "rate": 0 }, "mean": { "reward": 0.333333333333333, "inputTokens": 25135458.641414143, "outputTokens": 196677.30303030304 } } ], "safety": { "attemptCount": 905, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5626764360706693, "mean": { "reward": 0.5626764360706693, "costUsd": 8.817468730473037, "agentWallTimeSec": 2354.367686468646, "steps": 162.64576457645765, "inputTokens": 14201447.955995597, "outputTokens": 123276.32783278328, "cacheTokens": 13550056.765676552, "cacheHitRate": 0.938882976190201, "totalTokens": 14324724.283828393 }, "sums": { "costUsd": 2671.693025333334 }, "percentiles": { "costUsd": { "p05": 2.7192659427942796, "p25": 4.51524115951595, "p50": 6.599111777777779, "p75": 9.808614987898796, "p95": 18.236931686028605 }, "totalTokens": { "p05": 3709632.12431243, "p25": 6478872.97579758, "p50": 9948394.636963697, "p75": 15872444.876787692, "p95": 31362264.765786577 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "1.0.40", "dateReleased": null }, "max": { "version": "1.0.40", "dateReleased": null } } } }, { "id": "28ef65135f08f035dc93a97d324fea73", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "deepseek", "hostModelSlug": "deepseek_deepseek-v4-flash-0731", "display": { "agent": "Codex", "model": "DeepSeek V4 Flash 0731 (max)", "creator": { "agent": "OpenAI", "model": "DeepSeek" } }, "displayLabel": "Codex - DeepSeek V4 Flash 0731 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.542772861356932, "inputTokens": 18697668.702064898, "outputTokens": 97092.21238938053 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.513440860215054, "inputTokens": 8160643.36827957, "outputTokens": 35348.784946236556 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 21, "reviewedAttemptCount": 21, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 21 } ], "rate": 0 }, "mean": { "reward": 0.106060606060606, "inputTokens": 22757591.373737372, "outputTokens": 136457.94949494948 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.3874247758775307, "mean": { "reward": 0.3874247758775307, "costUsd": 0.08506419165060494, "agentWallTimeSec": 1097.0404400440034, "steps": 121.66666666666667, "inputTokens": 15269826.30913091, "outputTokens": 80398.9900990099, "cacheTokens": 15142136.818481851, "cacheHitRate": 0.9869993314104538, "totalTokens": 15350225.299229927 }, "sums": { "costUsd": 25.774450070133362 }, "percentiles": { "costUsd": { "p05": 0.034830933659581974, "p25": 0.054086775420022014, "p50": 0.07488192848580855, "p75": 0.09969517195863582, "p95": 0.17027440660206825 }, "totalTokens": { "p05": 3870194.438283829, "p25": 7621129.167766776, "p50": 11973775.719471961, "p75": 19587023.848184794, "p95": 38473133.53927391 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.2", "dateReleased": "2026-09-03" } }, "swe-atlas-qna": { "min": { "version": "0.146.0", "dateReleased": "2026-07-29" }, "max": { "version": "0.146.1", "dateReleased": "2026-08-05" } }, "terminal-bench-v4": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.2", "dateReleased": "2026-09-03" } } } }, { "id": "33e80f383dc644527161435b076798ad", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-high", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (high)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (high)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.666666666666667, "inputTokens": 2606937.696165192, "outputTokens": 46314.92330383481 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 31, "hardStopAttemptCount": 0, "recoveredAttemptCount": 31, "fallbackAttemptCount": 31, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.013440860215053764 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 37, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 83, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 96, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.08333333333333333 }, "rewardHacking": null, "mean": { "reward": 0.564516129032258, "inputTokens": 1796491.9838709678, "outputTokens": 20475.11559139785 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 2, "hardStopAttemptCount": 0, "recoveredAttemptCount": 2, "fallbackAttemptCount": 2, "refusals": [ { "turn": 92, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 128, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.010101010101010102 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 83, "reviewedAttemptCount": 83, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 83 } ], "rate": 0 }, "mean": { "reward": 0.419191919191919, "inputTokens": 4147711.272727273, "outputTokens": 72740.92424242424 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 33, "hardStopAttemptCount": 0, "recoveredAttemptCount": 33, "fallbackAttemptCount": 33, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.004480286738351254 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 37, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 83, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 96, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 92, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 128, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.031144781144781142 }, "indexScore": 0.5501249049636148, "mean": { "reward": 0.5501249049636148, "costUsd": 1.235497127172719, "agentWallTimeSec": 735.1809878987912, "steps": 36.772277227722775, "inputTokens": 2610884.190319032, "outputTokens": 41496.37513751376, "cacheTokens": 2507283.205720573, "cacheHitRate": 0.9477934370079523, "totalTokens": 2652380.565456549 }, "sums": { "costUsd": 374.355629533333 }, "percentiles": { "costUsd": { "p05": 0.26541014830583043, "p25": 0.45273301105610575, "p50": 0.8423270638063807, "p75": 1.452190312926292, "p95": 3.3796598031683143 }, "totalTokens": { "p05": 393734.5768976899, "p25": 771180.2568756866, "p50": 1799032.7986798703, "p75": 3205427.8019801965, "p95": 8253061.206600648 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "f2de07373c3626f1b9269a6ecad7804c", "isDefault": true, "isHighlighted": true, "agentName": "Muse Code", "provider": "meta", "hostModelSlug": "meta_aa_glacier135", "display": { "agent": "Muse Code", "model": "Muse Spark 1.3 (max)", "creator": { "agent": "Meta", "model": "Meta" } }, "displayLabel": "Muse Code - Muse Spark 1.3 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.716814159292036, "inputTokens": 13526615.77581121, "outputTokens": 122491.69026548673 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.594086021505376, "inputTokens": 4493624.8655913975, "outputTokens": 23615.916666666668 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 63, "reviewedAttemptCount": 63, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 63 } ], "rate": 0 }, "mean": { "reward": 0.318181818181818, "inputTokens": 43900184.35858586, "outputTokens": 955492.9924242424 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5430273329930766, "mean": { "reward": 0.5430273329930766, "costUsd": 3.977504096947204, "agentWallTimeSec": 1106.8533531353135, "steps": 185.23817381738175, "inputTokens": 16445971.068206819, "outputTokens": 263473.5055005501, "cacheTokens": 16090656.487898799, "cacheHitRate": 0.9712826195905685, "totalTokens": 16709444.57370737 }, "sums": { "costUsd": 1205.183741375 }, "percentiles": { "costUsd": { "p05": 0.6994728876952683, "p25": 1.0872893968371842, "p50": 2.0734771829482943, "p75": 3.7535131079207953, "p95": 13.168053740192518 }, "totalTokens": { "p05": 2516743.7783278334, "p25": 4610005.632013202, "p50": 9203495.684268419, "p75": 15732086.408140825, "p95": 62899130.50220029 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" }, "max": { "version": "1.0.3-R2198.1", "dateReleased": "2026-09-04" } } } }, { "id": "666605c66ad569c503ce53c01682e285", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-xhigh", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (xhigh)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.684365781710914, "inputTokens": 7430693.578171091, "outputTokens": 150014.38643067845 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 31, "hardStopAttemptCount": 0, "recoveredAttemptCount": 31, "fallbackAttemptCount": 31, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 115, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 162, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.08333333333333333 }, "rewardHacking": null, "mean": { "reward": 0.620967741935484, "inputTokens": 5431788.322580645, "outputTokens": 64841.01344086022 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 65, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.015151515151515152 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 115, "reviewedAttemptCount": 115, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 115 } ], "rate": 0 }, "mean": { "reward": 0.580808080808081, "inputTokens": 9034401.93939394, "outputTokens": 198393.7626262626 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 34, "hardStopAttemptCount": 0, "recoveredAttemptCount": 34, "fallbackAttemptCount": 34, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 26, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 115, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 162, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 65, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 70, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.032828282828282825 }, "indexScore": 0.628713868151493, "mean": { "reward": 0.628713868151493, "costUsd": 3.3333865348734886, "agentWallTimeSec": 1619.0134708470848, "steps": 78.31683168316832, "inputTokens": 6961982.357535753, "outputTokens": 125696.03850385037, "cacheTokens": 6693763.679867986, "cacheHitRate": 0.9574714756466342, "totalTokens": 7087678.3960396005 }, "sums": { "costUsd": 1010.0161200666669 }, "percentiles": { "costUsd": { "p05": 1.013542773971397, "p25": 1.7970568723322353, "p50": 2.8112455694169425, "p75": 4.45485277458746, "p95": 7.184486563465355 }, "totalTokens": { "p05": 1669169.926072607, "p25": 3466916.930693066, "p50": 5558230.887788775, "p75": 9190033.857535753, "p95": 17286164.792959306 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "ac8276cd5d15ac2daacb9956ad44c132", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-6-luna", "display": { "agent": "Codex", "model": "GPT-6 Luna (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6 Luna (max) ({'reasoning_effort': 'max'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 338, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.63716814159292, "inputTokens": 12385746.131268436, "outputTokens": 108271.5634218289 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.443548387096774, "inputTokens": 5786509.854838709, "outputTokens": 53341.779569892475 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 30, "reviewedAttemptCount": 30, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 30 } ], "rate": 0 }, "mean": { "reward": 0.151515151515152, "inputTokens": 14381227.393939395, "outputTokens": 165884.42424242425 } } ], "safety": { "attemptCount": 908, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4107438934016153, "mean": { "reward": 0.4107438934016153, "costUsd": 0.17591172304455457, "agentWallTimeSec": 1285.9268514851478, "steps": 88.64301430143014, "inputTokens": 10119727.864136413, "outputTokens": 98341.38393839383, "cacheTokens": 9897312.69911991, "cacheHitRate": 0.9606438143896164, "totalTokens": 10218069.248074815 }, "sums": { "costUsd": 53.3012520824999 }, "percentiles": { "costUsd": { "p05": 0.05139677199229924, "p25": 0.08709199411441132, "p50": 0.12790623996149636, "p75": 0.23731735201595155, "p95": 0.4026450882722773 }, "totalTokens": { "p05": 2000123.225962595, "p25": 3915894.559955995, "p50": 7023106.606160604, "p75": 13626608.82783279, "p95": 26835833.369636964 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "d1003684d8f29b95a3b14d2a7722221a", "isDefault": false, "isHighlighted": false, "agentName": "Antigravity SDK v0.1.12", "provider": "google", "hostModelSlug": "google_skimaki_ai-studio", "display": { "agent": "Antigravity SDK", "model": "Gemini 3.8 Flash (high)", "creator": { "agent": "Google", "model": "Google" } }, "displayLabel": "Antigravity SDK - Gemini 3.8 Flash (high)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.657817109144543, "inputTokens": 16400687.365781711, "outputTokens": 148259.0501474926 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.451612903225807, "inputTokens": 9020406.623655913, "outputTokens": 58109 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 33, "reviewedAttemptCount": 33, "flaggedAttemptCount": 4, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 33 } ], "rate": 0.020202020202020204 }, "mean": { "reward": 0.146464646464646, "inputTokens": 16468964.714646464, "outputTokens": 168736.55303030304 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4186315529449987, "mean": { "reward": 0.4186315529449987, "costUsd": 2.4673922004125437, "agentWallTimeSec": 702.7110253025301, "steps": 188.4884488448845, "inputTokens": 13395246.748624865, "outputTokens": 115826.40649064905, "cacheTokens": 11871691.682068229, "cacheHitRate": 0.8697428234581179, "totalTokens": 13511073.155115515 }, "sums": { "costUsd": 747.619836725 }, "percentiles": { "costUsd": { "p05": 1.099103153622112, "p25": 1.6439984646864687, "p50": 2.1229609128712874, "p75": 3.008778207632013, "p95": 4.849891568358086 }, "totalTokens": { "p05": 4679184.810451044, "p25": 8015173.094609471, "p50": 11559537.268426849, "p75": 16883320.984598454, "p95": 28318875.752915297 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.1.16", "dateReleased": "2026-09-02" }, "max": { "version": "0.1.16", "dateReleased": "2026-09-02" } }, "swe-atlas-qna": { "min": { "version": "0.1.12", "dateReleased": "2026-08-13" }, "max": { "version": "0.1.12", "dateReleased": "2026-08-13" } }, "terminal-bench-v4": { "min": { "version": "0.1.16", "dateReleased": "2026-09-02" }, "max": { "version": "0.1.16", "dateReleased": "2026-09-02" } } } }, { "id": "ef63fafa9b4a635522183c879ae8435b", "isDefault": true, "isHighlighted": true, "agentName": "Antigravity CLI", "provider": "google", "hostModelSlug": "google_barium-bb_ai-studio", "display": { "agent": "Antigravity CLI", "model": "Gemini 4 Argon", "creator": { "agent": "Google", "model": "Google" } }, "displayLabel": "Antigravity CLI - Gemini 4 Argon (high)", "variantOf": null, "isUnavailable": true, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 336, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.787610619469027, "inputTokens": 19901657.223214287, "outputTokens": 152422.96130952382 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 371, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.564516129032258, "inputTokens": 7075681.379032258, "outputTokens": 48670.67473118279 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 112, "reviewedAttemptCount": 112, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 112 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.560606060606061, "inputTokens": 15263258.186868686, "outputTokens": 162940.09595959596 } } ], "safety": { "attemptCount": 905, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.637577603035782, "mean": { "reward": 0.637577603035782, "costUsd": 5.8439053894448145, "agentWallTimeSec": 2070.1686032008843, "steps": 149.1832229580574, "inputTokens": 13621672.650110375, "outputTokens": 112121.13134657839, "cacheTokens": 11850855.205596024, "cacheHitRate": 0.8700000000000004, "totalTokens": 13733793.781456953 }, "sums": { "costUsd": 1764.859427612334 }, "percentiles": { "costUsd": { "p05": 1.3165824554546348, "p25": 2.7006862521677717, "p50": 4.259847111864239, "p75": 6.94478139742605, "p95": 15.23551975602052 }, "totalTokens": { "p05": 2407380.2615894047, "p25": 5633674.4067328945, "p50": 9566161.003311247, "p75": 16754521.037527593, "p95": 37940692.606512114 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.2.10", "dateReleased": null }, "max": { "version": "1.2.10", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "1.2.10", "dateReleased": null }, "max": { "version": "1.2.10", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "1.2.10", "dateReleased": null }, "max": { "version": "1.2.10", "dateReleased": null } } } }, { "id": "fa2595093fa640e73d072e7d1aa7df82", "isDefault": true, "isHighlighted": false, "agentName": "Devin Fusion CLI", "provider": "cognition", "hostModelSlug": "cognition_fusion-gpt-6-astra-xhigh-sidekick-penguin-medium", "display": { "agent": "Devin Fusion CLI", "model": "GPT-6 Astra XHigh + SWE-2 Medium" }, "displayLabel": "Devin Fusion CLI - GPT-6 Astra XHigh + SWE-2 Medium", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 1, "hardStopAttemptCount": 0, "recoveredAttemptCount": 1, "fallbackAttemptCount": 1, "refusals": [ { "turn": 5, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 } ], "rate": 0.0029498525073746312 }, "rewardHacking": null, "mean": { "reward": 0.672566371681416, "inputTokens": 10312445.209439527, "outputTokens": 83838.11799410029 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 25, "hardStopAttemptCount": 1, "recoveredAttemptCount": 24, "fallbackAttemptCount": 24, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.013440860215053764 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 1, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 10, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.06720430107526881 }, "rewardHacking": null, "mean": { "reward": 0.594086021505376, "inputTokens": 2112115.313172043, "outputTokens": 23623.837365591397 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 99, "reviewedAttemptCount": 99, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 99 } ], "rate": 0 }, "mean": { "reward": 0.5, "inputTokens": 6001673.575757576, "outputTokens": 71827.65656565657 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 26, "hardStopAttemptCount": 1, "recoveredAttemptCount": 25, "fallbackAttemptCount": 25, "refusals": [ { "turn": 5, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 5, "outcome": "fallback", "rate": 0.004480286738351254 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 1, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 10, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [ { "model": "gpt-5-6-sol-xhigh", "releaseSlug": "gpt-5-6-sol", "releaseName": "GPT-5.6 Sol", "releaseDate": "2026-07-09", "creatorSlug": "openai" }, { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-high", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.023384717860881147 }, "indexScore": 0.588884131062264, "mean": { "reward": 0.588884131062264, "costUsd": 4.536492777821779, "agentWallTimeSec": 1481.89598019802, "steps": 78.55115511551155, "inputTokens": 6017554.665016501, "outputTokens": 56579.83003300329, "cacheTokens": 5695796.99229923, "cacheHitRate": 0.920061132172241, "totalTokens": 6074134.4950494915 }, "sums": { "costUsd": 1374.557311679999 }, "percentiles": { "costUsd": { "p05": 1.8884645795379544, "p25": 2.7901938516089118, "p50": 3.6622604586358625, "p75": 5.514399754826733, "p95": 9.903196057832783 }, "totalTokens": { "p05": 1278837.6278327832, "p25": 2419160.8525852584, "p50": 3840277.500550057, "p75": 7328820.564356423, "p95": 15347920.93509351 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } } } }, { "id": "a2c87c062f3cef73d8525e7578f14742", "isDefault": true, "isHighlighted": true, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-max", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (max)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.71976401179941, "inputTokens": 22525711.609144542, "outputTokens": 614455.6474926254 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 39, "hardStopAttemptCount": 0, "recoveredAttemptCount": 39, "fallbackAttemptCount": 39, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 77, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 10, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 20, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 23, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 29, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 64, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 67, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 97, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 108, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 139, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 147, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 185, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 243, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 254, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 353, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 593, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.10483870967741936 }, "rewardHacking": null, "mean": { "reward": 0.669354838709677, "inputTokens": 34503496.576612905, "outputTokens": 552623.7553763441 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 6, "hardStopAttemptCount": 0, "recoveredAttemptCount": 6, "fallbackAttemptCount": 6, "refusals": [ { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 81, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 87, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 109, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.030303030303030304 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 131, "reviewedAttemptCount": 131, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 131 } ], "rate": 0 }, "mean": { "reward": 0.661616161616162, "inputTokens": 21212057.686868686, "outputTokens": 668705.3535353536 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 45, "hardStopAttemptCount": 0, "recoveredAttemptCount": 45, "fallbackAttemptCount": 45, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 77, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 10, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 16, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 20, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 23, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 29, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 60, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 64, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 67, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 69, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 97, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 108, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 139, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 147, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 185, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 243, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 254, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 353, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 593, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 53, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 81, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 87, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 109, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.045047246660149885 }, "indexScore": 0.6835783373750831, "mean": { "reward": 0.6835783373750831, "costUsd": 14.190800796369649, "agentWallTimeSec": 5244.965372937293, "steps": 265.93069306930687, "inputTokens": 27141368.959295932, "outputTokens": 600968.2744774478, "cacheTokens": 26006027.66886688, "cacheHitRate": 0.9506214943317096, "totalTokens": 27742337.233773403 }, "sums": { "costUsd": 4299.812641299994 }, "percentiles": { "costUsd": { "p05": 4.66515082839384, "p25": 7.936658873542355, "p50": 12.34840697040706, "p75": 19.223640478410346, "p95": 28.29229594782175 }, "totalTokens": { "p05": 6340397.848074809, "p25": 13743009.44829484, "p50": 23752423.691969212, "p75": 37935915.66336632, "p95": 59829310.28954897 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "cd4c865dc2f539b650d741fd2ebf7f2d", "isDefault": true, "isHighlighted": false, "agentName": "Claude Code", "provider": "alibaba_cloud", "hostModelSlug": "alibaba_cloud_qwen3-8-max-public", "display": { "agent": "Claude Code", "model": "Qwen3.8 Max", "creator": { "agent": "Anthropic", "model": "Alibaba Cloud" } }, "displayLabel": "Claude Code - Qwen3.8 Max", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.510324483775811, "inputTokens": 10307762.331858408, "outputTokens": 104418.39233038347 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.620967741935484, "inputTokens": 5159227.653225807, "outputTokens": 45811.73655913978 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 33, "reviewedAttemptCount": 33, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 33 } ], "rate": 0 }, "mean": { "reward": 0.166666666666667, "inputTokens": 12143161.925641026, "outputTokens": 223315.56923076924 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.43265296412598736, "mean": { "reward": 0.43265296412598736, "costUsd": 3.478965577837015, "agentWallTimeSec": 3784.416073707372, "steps": 110.03670136244394, "inputTokens": 8600561.252779894, "outputTokens": 106332.54533299482, "cacheTokens": 8290829.75631717, "cacheHitRate": 0.9507113544554301, "totalTokens": 8706893.798112892 }, "sums": { "costUsd": 1048.213634833334 }, "percentiles": { "costUsd": { "p05": 1.2462484540154002, "p25": 1.9738426430143006, "p50": 2.838224532178221, "p75": 4.649543275302529, "p95": 7.302521150137508 }, "totalTokens": { "p05": 2586103.0433443333, "p25": 4938223.044004401, "p50": 7296928.9218921885, "p75": 11298465.588558838, "p95": 18720391.11265126 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.261", "dateReleased": "2026-09-04" }, "max": { "version": "2.1.261", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "2.1.224", "dateReleased": "2026-08-07" }, "max": { "version": "2.1.236", "dateReleased": "2026-08-19" } }, "terminal-bench-v4": { "min": { "version": "2.1.261", "dateReleased": "2026-09-04" }, "max": { "version": "2.1.261", "dateReleased": "2026-09-04" } } } }, { "id": "34970c963bdfd318e46205514d46ea4f", "isDefault": true, "isHighlighted": false, "agentName": "Devin Fusion CLI", "provider": "cognition", "hostModelSlug": "cognition_fusion-claude-fable-5-1-xhigh-sidekick-penguin-medium", "display": { "agent": "Devin Fusion CLI", "model": "Claude Fable 5.1 XHigh + SWE-2 Medium" }, "displayLabel": "Devin Fusion CLI - Claude Fable 5.1 XHigh + SWE-2 Medium", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 1, "hardStopAttemptCount": 0, "recoveredAttemptCount": 1, "fallbackAttemptCount": 1, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 } ], "rate": 0.0029498525073746312 }, "rewardHacking": null, "mean": { "reward": 0.631268436578171, "inputTokens": 10080280.292035399, "outputTokens": 111599.9557522124 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 42, "hardStopAttemptCount": 0, "recoveredAttemptCount": 42, "fallbackAttemptCount": 42, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 21, "outcome": "fallback", "rate": 0.056451612903225805 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 22, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 36, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 37, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 60, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.11290322580645161 }, "rewardHacking": null, "mean": { "reward": 0.658602150537635, "inputTokens": 5492119.514784946, "outputTokens": 55827.5685483871 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 197, "refusedAttemptCount": 19, "hardStopAttemptCount": 0, "recoveredAttemptCount": 19, "fallbackAttemptCount": 19, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.030456852791878174 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.01015228426395939 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 6, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 8, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 13, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 25, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 } ], "rate": 0.09644670050761421 }, "rewardHacking": { "retainedAttemptCount": 197, "passingAttemptCount": 111, "reviewedAttemptCount": 111, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 111 } ], "rate": 0 }, "mean": { "reward": 0.560606060606061, "inputTokens": 16451944.767676767, "outputTokens": 167921.75252525252 } } ], "safety": { "attemptCount": 908, "refusedAttemptCount": 62, "hardStopAttemptCount": 0, "recoveredAttemptCount": 62, "fallbackAttemptCount": 62, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 21, "outcome": "fallback", "rate": 0.01881720430107527 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 22, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 36, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 37, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 55, "chain": [ { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 60, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.010152284263959392 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.00338409475465313 }, { "turn": 2, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 4, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" }, { "model": "claude-opus-4-8-xhigh", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 5, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 6, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 7, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 8, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 13, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 25, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 27, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 63, "chain": [ { "model": "claude-opus-5-xhigh", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 } ], "rate": 0.07076659294048016 }, "indexScore": 0.6168255492406223, "mean": { "reward": 0.6168255492406223, "costUsd": 7.897126985217273, "agentWallTimeSec": 2146.9920896589642, "steps": 99.53135313531354, "inputTokens": 9590504.44719472, "outputTokens": 101043.72662266226, "cacheTokens": 9279592.760176014, "cacheHitRate": 0.9584900963367992, "totalTokens": 9691548.173817372 }, "sums": { "costUsd": 2392.829476520833 }, "percentiles": { "costUsd": { "p05": 3.1036853524587458, "p25": 4.744201155569304, "p50": 6.569551072249726, "p75": 9.734336789150163, "p95": 16.72564247919966 }, "totalTokens": { "p05": 1840241.9694169401, "p25": 3708383.8074807483, "p50": 6571844.10121012, "p75": 12694519.213971417, "p95": 27823135.185038492 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "3000.10.7", "dateReleased": null }, "max": { "version": "3000.10.7", "dateReleased": null } } } }, { "id": "709f801275dedae5675b1f56e97c9df3", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-5-6-sol", "display": { "agent": "Codex", "model": "GPT-5.6 Sol (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-5.6 Sol (max) ({'reasoning_effort': 'max'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.722713864306785, "inputTokens": 10907777.09439528, "outputTokens": 56943.890855457226 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 1, "hardStopAttemptCount": 1, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.002688172043010753 }, "rewardHacking": null, "mean": { "reward": 0.540322580645161, "inputTokens": 7898018.697580645, "outputTokens": 31499.083333333332 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 74, "reviewedAttemptCount": 74, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 74 } ], "rate": 0 }, "mean": { "reward": 0.373737373737374, "inputTokens": 12950366.515151516, "outputTokens": 68992.93181818182 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 1, "hardStopAttemptCount": 1, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.000896057347670251 }, "indexScore": 0.54559127289644, "mean": { "reward": 0.54559127289644, "costUsd": 6.353318230693071, "agentWallTimeSec": 1237.696445544555, "steps": 116.39493949394938, "inputTokens": 10120981.254675467, "outputTokens": 49155.37788778878, "cacheTokens": 9784083.974697473, "cacheHitRate": 0.9458037017976718, "totalTokens": 10170136.632563243 }, "sums": { "costUsd": 1925.0554239 }, "percentiles": { "costUsd": { "p05": 1.8512760725412558, "p25": 3.1885388294829466, "p50": 5.214009559955996, "p75": 7.762229316171614, "p95": 14.896605548536868 }, "totalTokens": { "p05": 1994550.4016501652, "p25": 3937210.7585258516, "p50": 7714596.468646868, "p75": 12446152.79537957, "p95": 27006730.063476328 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.139.0", "dateReleased": "2026-06-09" }, "max": { "version": "0.147.0", "dateReleased": "2026-08-07" } }, "terminal-bench-v4": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, { "id": "93acd59f31f8e5478686d650aa650f79", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (medium)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (medium) ({'reasoning_effort': 'medium'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.71976401179941, "inputTokens": 1875718.3569321535, "outputTokens": 22954.34808259587 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 12, "hardStopAttemptCount": 12, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 10, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03225806451612903 }, "rewardHacking": null, "mean": { "reward": 0.60752688172043, "inputTokens": 1970188.7650273223, "outputTokens": 13754.166666666666 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 102, "reviewedAttemptCount": 102, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 102 } ], "rate": 0 }, "mean": { "reward": 0.515151515151515, "inputTokens": 3535272.3131313133, "outputTokens": 37750.72727272727 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 12, "hardStopAttemptCount": 12, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 10, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.01075268817204301 }, "indexScore": 0.6141474695571183, "mean": { "reward": 0.6141474695571183, "costUsd": 0.7046746644898908, "agentWallTimeSec": 651.3228668866898, "steps": 37.98899889988999, "inputTokens": 2275866.5144006205, "outputTokens": 22412.23102310231, "cacheTokens": 2176436.9277894986, "cacheHitRate": 0.9239660644436648, "totalTokens": 2298278.745423723 }, "sums": { "costUsd": 212.41134426666656 }, "percentiles": { "costUsd": { "p05": 0.21984839405940593, "p25": 0.3595459733773378, "p50": 0.4937551212321231, "p75": 0.7913959203520352, "p95": 1.468924121518151 }, "totalTokens": { "p05": 376691.5391639165, "p25": 783633.2106710669, "p50": 1298877.5742574276, "p75": 2736930.987898792, "p95": 5339040.968976896 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "ea296ce785422d3b97cfff4427f2e0d2", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.696165191740413, "inputTokens": 3676096.7050147494, "outputTokens": 70758.95280235988 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 15, "hardStopAttemptCount": 15, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 25, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 23, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 32, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 65, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.04032258064516129 }, "rewardHacking": null, "mean": { "reward": 0.577956989247312, "inputTokens": 3171478.0546448086, "outputTokens": 35668.81420765027 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 105, "reviewedAttemptCount": 105, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 105 } ], "rate": 0 }, "mean": { "reward": 0.53030303030303, "inputTokens": 5825821.813131313, "outputTokens": 94906.77777777778 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 15, "hardStopAttemptCount": 15, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 25, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 23, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 32, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 65, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.013440860215053764 }, "indexScore": 0.6014750704302517, "mean": { "reward": 0.6014750704302517, "costUsd": 1.554119528390054, "agentWallTimeSec": 1463.0464224422437, "steps": 40.45654565456545, "inputTokens": 3937843.0564663024, "outputTokens": 61658.55432920342, "cacheTokens": 3740552.1873072563, "cacheHitRate": 0.9281954641245814, "totalTokens": 3999501.6107955067 }, "sums": { "costUsd": 468.718937949999 }, "percentiles": { "costUsd": { "p05": 0.50620098809681, "p25": 0.7844812155115526, "p50": 1.140491010561054, "p75": 1.8937155413641367, "p95": 3.7431764045654536 }, "totalTokens": { "p05": 795880.5675467533, "p25": 1648548.6529152917, "p50": 2724214.3113311334, "p75": 4694077.841584162, "p95": 10822464.925852576 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "409518818f1e9ad21995d70631ab05b6", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-opus-5", "display": { "agent": "Claude Code", "model": "Opus 5 (max)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Opus 5 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.625368731563422, "inputTokens": 13358102.041297935, "outputTokens": 129793.94690265486 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.620967741935484, "inputTokens": 7586223.970430108, "outputTokens": 61960.48924731183 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 197, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 }, { "turn": 59, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005076142131979695 } ], "rate": 0.015228426395939087 }, "rewardHacking": { "retainedAttemptCount": 197, "passingAttemptCount": 108, "reviewedAttemptCount": 108, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 108 } ], "rate": 0 }, "mean": { "reward": 0.545454545454545, "inputTokens": 14469933.381313132, "outputTokens": 288742.0505050505 } } ], "safety": { "attemptCount": 908, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 42, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 }, { "turn": 59, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.001692047377326565 } ], "rate": 0.005076142131979696 }, "indexScore": 0.5972636729844837, "mean": { "reward": 0.5972636729844837, "costUsd": 10.789154699257413, "agentWallTimeSec": 2515.540235973597, "steps": 152.13036303630363, "inputTokens": 11238194.40979098, "outputTokens": 136656.0792079208, "cacheTokens": 10929014.04565457, "cacheHitRate": 0.9698258997320075, "totalTokens": 11374850.488998907 }, "sums": { "costUsd": 3269.1138738750005 }, "percentiles": { "costUsd": { "p05": 4.070634550165014, "p25": 6.0050860609185905, "p50": 8.377338476347639, "p75": 13.669421935368524, "p95": 24.306012511496142 }, "totalTokens": { "p05": 3637409.260286029, "p25": 6155919.92079208, "p50": 9436168.620462047, "p75": 14396226.760176001, "p95": 24864036.97546756 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } }, "swe-atlas-qna": { "min": { "version": "2.1.218", "dateReleased": "2026-07-22" }, "max": { "version": "2.1.227", "dateReleased": "2026-08-10" } }, "terminal-bench-v4": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } } } }, { "id": "e9f9d3ae5b03aa060114dd4b3275d67d", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-low", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (low)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (low)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.619469026548673, "inputTokens": 831380.2064896756, "outputTokens": 16103.77581120944 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 15, "hardStopAttemptCount": 0, "recoveredAttemptCount": 15, "fallbackAttemptCount": 15, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 4, "outcome": "fallback", "rate": 0.010752688172043012 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.04032258064516129 }, "rewardHacking": null, "mean": { "reward": 0.389784946236559, "inputTokens": 393558.31720430107, "outputTokens": 6681.293010752688 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 3, "hardStopAttemptCount": 0, "recoveredAttemptCount": 3, "fallbackAttemptCount": 3, "refusals": [ { "turn": 138, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 183, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 365, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.015151515151515152 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 50, "reviewedAttemptCount": 50, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 50 } ], "rate": 0 }, "mean": { "reward": 0.252525252525253, "inputTokens": 2251236.696969697, "outputTokens": 32428.20707070707 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 18, "hardStopAttemptCount": 0, "recoveredAttemptCount": 18, "fallbackAttemptCount": 18, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 4, "outcome": "fallback", "rate": 0.003584229390681004 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 138, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 183, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 365, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.018491365265558814 }, "indexScore": 0.420593075103495, "mean": { "reward": 0.420593075103495, "costUsd": 0.48311011474147336, "agentWallTimeSec": 380.85617931793183, "steps": 18.578657865786578, "inputTokens": 961481.2431243125, "outputTokens": 15803.526952695267, "cacheTokens": 917182.5137513754, "cacheHitRate": 0.9307814644010347, "totalTokens": 977284.7700770069 }, "sums": { "costUsd": 146.3823647666666 }, "percentiles": { "costUsd": { "p05": 0.10326793619361947, "p25": 0.15142615379537977, "p50": 0.2763223977997799, "p75": 0.4517722253575359, "p95": 1.6299054529482935 }, "totalTokens": { "p05": 152419.26897689793, "p25": 250398.5033003298, "p50": 484363.37403740385, "p75": 929714.8008800887, "p95": 3672521.961386143 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "4d6aa9e9c8df931d54dba5956d94973c", "isDefault": false, "isHighlighted": false, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "private-anthropic_claude-beignet-eap-medium", "display": { "agent": "Claude Code", "model": "Sonnet 5.5 (medium)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Sonnet 5.5 (medium)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.654867256637168, "inputTokens": 1089240.6017699116, "outputTokens": 21123.365781710916 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 17, "hardStopAttemptCount": 0, "recoveredAttemptCount": 17, "fallbackAttemptCount": 17, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.008064516129032258 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.0456989247311828 }, "rewardHacking": null, "mean": { "reward": 0.448924731182796, "inputTokens": 567843.747311828, "outputTokens": 9186.077956989247 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 2, "hardStopAttemptCount": 0, "recoveredAttemptCount": 2, "fallbackAttemptCount": 2, "refusals": [ { "turn": 75, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 144, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.010101010101010102 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 54, "reviewedAttemptCount": 54, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 54 } ], "rate": 0 }, "mean": { "reward": 0.272727272727273, "inputTokens": 2728043.8636363638, "outputTokens": 40784.555555555555 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 19, "hardStopAttemptCount": 0, "recoveredAttemptCount": 19, "fallbackAttemptCount": 19, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 7, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.0026881720430107525 }, { "turn": 1, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 24, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 75, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 144, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.018599978277397632 }, "indexScore": 0.4588397535157457, "mean": { "reward": 0.4588397535157457, "costUsd": 0.6187507753025301, "agentWallTimeSec": 510.7721826182621, "steps": 22.20902090209021, "inputTokens": 1232830.7183718372, "outputTokens": 20520.774477447743, "cacheTokens": 1173805.1144114407, "cacheHitRate": 0.9341009883549257, "totalTokens": 1253351.4928492843 }, "sums": { "costUsd": 187.48148491666672 }, "percentiles": { "costUsd": { "p05": 0.1301400262926294, "p25": 0.2095640195269526, "p50": 0.41369848030803086, "p75": 0.7082445721672166, "p95": 2.060476164268427 }, "totalTokens": { "p05": 194819.09009900966, "p25": 320817.6897689771, "p50": 727342.763476347, "p75": 1444183.7959295923, "p95": 4695596.860066004 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "7a0dfd2a7a9d93056f860d4675586d66", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-6-sol", "display": { "agent": "Codex", "model": "GPT-6 Sol (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6 Sol (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 337, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.690265486725664, "inputTokens": 9754196.967551623, "outputTokens": 70300.33038348083 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.575268817204301, "inputTokens": 6133614.827956989, "outputTokens": 37933.720430107525 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 87, "reviewedAttemptCount": 87, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 87 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.434343434343434, "inputTokens": 16668544.737373738, "outputTokens": 93810.5303030303 } } ], "safety": { "attemptCount": 907, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5666259127577997, "mean": { "reward": 0.5666259127577997, "costUsd": 2.989729004180416, "agentWallTimeSec": 1338.0527051705185, "steps": 86.55005500550054, "inputTokens": 9778602.140814083, "outputTokens": 62175.62266226623, "cacheTokens": 9598928.887788778, "cacheHitRate": 0.9640119921972632, "totalTokens": 9840777.763476355 }, "sums": { "costUsd": 905.887888266666 }, "percentiles": { "costUsd": { "p05": 0.8303700961936205, "p25": 1.412352845434542, "p50": 2.0778670118811906, "p75": 3.503332819251925, "p95": 8.13729606064907 }, "totalTokens": { "p05": 1633543.1261826195, "p25": 3470166.2843784397, "p50": 5870785.018701871, "p75": 11366503.224422423, "p95": 31676596.306490637 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "4aee8aa2d4a0bad6068644f40edae235", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (high)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (high) ({'reasoning_effort': 'high'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.705014749262537, "inputTokens": 2382530.2064896757, "outputTokens": 31206.94395280236 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 14, "hardStopAttemptCount": 14, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 7, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 39, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03763440860215054 }, "rewardHacking": null, "mean": { "reward": 0.599462365591398, "inputTokens": 2421836.2786885244, "outputTokens": 17768.75956284153 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 1, "hardStopAttemptCount": 1, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.005050505050505051 } ], "rate": 0.005050505050505051 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 99, "reviewedAttemptCount": 99, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 99 } ], "rate": 0 }, "mean": { "reward": 0.5, "inputTokens": 4184679.611111111, "outputTokens": 45169.030303030304 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 15, "hardStopAttemptCount": 15, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 7, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 39, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.0016835016835016836 } ], "rate": 0.014228304550885196 }, "indexScore": 0.6014923716179784, "mean": { "reward": 0.6014923716179784, "costUsd": 0.8892413135295484, "agentWallTimeSec": 799.7652123212325, "steps": 38.97909790979098, "inputTokens": 2791163.2548648305, "outputTokens": 28748.735486663423, "cacheTokens": 2665415.3918375443, "cacheHitRate": 0.9278388366571141, "totalTokens": 2819911.9903514963 }, "sums": { "costUsd": 268.0356914333333 }, "percentiles": { "costUsd": { "p05": 0.312068081210121, "p25": 0.4989411210121013, "p50": 0.6891893533553354, "p75": 0.9873919497799788, "p95": 2.1023536615841585 }, "totalTokens": { "p05": 527228.1344334432, "p25": 1136802.8481848184, "p50": 1870636.3718371852, "p75": 3261089.490649065, "p95": 6454477.248184814 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "9e810ff3c6cce5ebaaff7c744c687a47", "isDefault": true, "isHighlighted": true, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_vega-alpha", "display": { "agent": "Codex", "model": "GPT-6 Astra (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6 Astra (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.675516224188791, "inputTokens": 3338101.7050147494, "outputTokens": 56420.18289085546 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 19, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 8, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 14, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 16, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03494623655913978 }, "rewardHacking": null, "mean": { "reward": 0.618279569892473, "inputTokens": 2623791.1885245903, "outputTokens": 28534.494535519127 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 110, "reviewedAttemptCount": 110, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 110 } ], "rate": 0 }, "mean": { "reward": 0.555555555555555, "inputTokens": 4503151.8838383835, "outputTokens": 65844.76262626263 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 19, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 8, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 14, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 16, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.01164874551971326 }, "indexScore": 0.6164504498789397, "mean": { "reward": 0.6164504498789397, "costUsd": 7.46709735138597, "agentWallTimeSec": 1762.0330264026397, "steps": 39.378437843784376, "inputTokens": 3299549.915435806, "outputTokens": 47061.09677361179, "cacheTokens": 3141588.9050659174, "cacheHitRate": 0.9335591564904552, "totalTokens": 3346611.012209417 }, "sums": { "costUsd": 2251.127294666667 }, "percentiles": { "costUsd": { "p05": 2.7390315410891106, "p25": 4.340413036578659, "p50": 6.125789602860287, "p75": 8.728907821232124, "p95": 14.871133791914197 }, "totalTokens": { "p05": 824190.9740374036, "p25": 1550558.2557755776, "p50": 2403033.1683168346, "p75": 3942392.7057205727, "p95": 7480476.901430149 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.151.0", "dateReleased": "2026-08-29" }, "max": { "version": "0.151.0", "dateReleased": "2026-08-29" } }, "terminal-bench-v4": { "min": { "version": "0.153.4", "dateReleased": "2026-09-04" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, { "id": "fa589dad83ffcdd6cce72a467e552063", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (low)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (low) ({'reasoning_effort': 'low'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.675516224188791, "inputTokens": 1135333.3510324485, "outputTokens": 12865.970501474927 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 9, "hardStopAttemptCount": 9, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 4, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.024193548387096774 }, "rewardHacking": null, "mean": { "reward": 0.551075268817204, "inputTokens": 1278601.31147541, "outputTokens": 9084.045081967213 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 98, "reviewedAttemptCount": 98, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 98 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.48989898989899, "inputTokens": 3454474.217171717, "outputTokens": 28368.58080808081 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 9, "hardStopAttemptCount": 9, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 4, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 1, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.008064516129032258 }, "indexScore": 0.5721634943016617, "mean": { "reward": 0.5721634943016617, "costUsd": 0.49880364510270725, "agentWallTimeSec": 517.8138943894386, "steps": 35.74037403740374, "inputTokens": 1699123.8601417518, "outputTokens": 14695.05805334632, "cacheTokens": 1627295.6717524226, "cacheHitRate": 0.9135762473536158, "totalTokens": 1713818.9181950986 }, "sums": { "costUsd": 150.4073521333333 }, "percentiles": { "costUsd": { "p05": 0.14515691546754672, "p25": 0.2084983163916393, "p50": 0.2965749825082511, "p75": 0.5361287875687569, "p95": 1.465534774675468 }, "totalTokens": { "p05": 240942.00825082537, "p25": 418718.0632563254, "p50": 753671.7865786579, "p75": 1840610.7161716148, "p95": 5662162.880087997 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "55b029d6971b4d378335bb168eb98b3b", "isDefault": true, "isHighlighted": false, "agentName": "Kimi Code CLI", "provider": "moonshotai", "hostModelSlug": "moonshot_kimi-k3", "display": { "agent": "Kimi Code CLI", "model": "Kimi K3", "creator": { "agent": "Moonshot AI", "model": "Moonshot AI" } }, "displayLabel": "Kimi Code CLI - Kimi K3", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.684365781710915, "inputTokens": 8571957.041297935, "outputTokens": 75080.34808259587 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.661290322580645, "inputTokens": 4719526.534946237, "outputTokens": 49977.29838709677 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 42, "reviewedAttemptCount": 42, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 42 } ], "rate": 0 }, "mean": { "reward": 0.212121212121212, "inputTokens": 14964824.787878787, "outputTokens": 205571.03535353535 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.519259105470924, "mean": { "reward": 0.519259105470924, "costUsd": 5.047195443300332, "agentWallTimeSec": 3689.3441947194724, "steps": 173.36028602860284, "inputTokens": 8387890.666666667, "outputTokens": 93230.86688668867, "cacheTokens": 7968496.133333323, "cacheHitRate": 0.9500000000000017, "totalTokens": 8481121.533553353 }, "sums": { "costUsd": 1529.30021932 }, "percentiles": { "costUsd": { "p05": 1.1201608924026403, "p25": 2.1660128471864684, "p50": 4.013249413745875, "p75": 6.4578525155775575, "p95": 11.098114784331683 }, "totalTokens": { "p05": 1514640.538393838, "p25": 3247568.457095711, "p50": 6455899.625962595, "p75": 11496187.961496139, "p95": 19359817.106380656 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.36.0", "dateReleased": "2026-08-13" }, "max": { "version": "0.36.1", "dateReleased": "2026-08-14" } }, "swe-atlas-qna": { "min": { "version": "0.26.0", "dateReleased": "2026-07-16" }, "max": { "version": "0.26.0", "dateReleased": "2026-07-16" } }, "terminal-bench-v4": { "min": { "version": "0.36.1", "dateReleased": "2026-08-14" }, "max": { "version": "0.36.1", "dateReleased": "2026-08-14" } } } }, { "id": "29a1cdad0d140781d05b9b4746cfdd0d", "isDefault": true, "isHighlighted": true, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-fable-5-1", "display": { "agent": "Claude Code", "model": "Fable 5.1 (max) (with fallback)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Fable 5.1 (max) (with fallback)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.64306784660767, "inputTokens": 6428754.772861357, "outputTokens": 155542.35398230088 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 46, "hardStopAttemptCount": 0, "recoveredAttemptCount": 46, "fallbackAttemptCount": 46, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 31, "outcome": "fallback", "rate": 0.08333333333333333 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 40, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 112, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.12365591397849462 }, "rewardHacking": null, "mean": { "reward": 0.647849462365591, "inputTokens": 3623187.8387096776, "outputTokens": 81143.10752688172 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 28, "hardStopAttemptCount": 0, "recoveredAttemptCount": 28, "fallbackAttemptCount": 28, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.030303030303030304 }, { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.010101010101010102 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 5, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 31, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.1414141414141414 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 114, "reviewedAttemptCount": 114, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 114 } ], "rate": 0 }, "mean": { "reward": 0.575757575757576, "inputTokens": 7810711.565656566, "outputTokens": 194732.47474747474 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 74, "hardStopAttemptCount": 0, "recoveredAttemptCount": 74, "fallbackAttemptCount": 74, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 31, "outcome": "fallback", "rate": 0.027777777777777776 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 12, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 22, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 34, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 40, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 47, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 101, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 112, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 6, "outcome": "fallback", "rate": 0.010101010101010102 }, { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 8, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.0033670033670033673 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 5, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 11, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 27, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 31, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 38, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.08835668513087869 }, "indexScore": 0.6222249615769457, "mean": { "reward": 0.6222249615769457, "costUsd": 12.388092375137534, "agentWallTimeSec": 2089.900125412543, "steps": 36.65676567656766, "inputTokens": 5581622.259625964, "outputTokens": 133631.59955995603, "cacheTokens": 5205555.651265128, "cacheHitRate": 0.9159555838702423, "totalTokens": 5715253.85918592 }, "sums": { "costUsd": 3753.591989666663 }, "percentiles": { "costUsd": { "p05": 5.444339298624863, "p25": 7.8804961398515, "p50": 11.040917095709577, "p75": 14.93645990085259, "p95": 22.439563513833875 }, "totalTokens": { "p05": 1605004.4331133126, "p25": 2728342.3767876793, "p50": 4432038.257425743, "p75": 7298906.072607261, "p95": 13976824.222882291 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } }, "swe-atlas-qna": { "min": { "version": "2.1.259", "dateReleased": "2026-09-02" }, "max": { "version": "2.1.259", "dateReleased": "2026-09-02" } }, "terminal-bench-v4": { "min": { "version": "2.1.263", "dateReleased": "2026-09-06" }, "max": { "version": "2.1.263", "dateReleased": "2026-09-06" } } } }, { "id": "ab5074e404be48bbd4ac3c488ad9a3e2", "isDefault": true, "isHighlighted": true, "agentName": "Claude Code", "provider": "anthropic", "hostModelSlug": "anthropic_claude-opus-5-5", "display": { "agent": "Claude Code", "model": "Opus 5.5 (max)", "creator": { "agent": "Anthropic", "model": "Anthropic" } }, "displayLabel": "Claude Code - Opus 5.5 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 2, "hardStopAttemptCount": 0, "recoveredAttemptCount": 2, "fallbackAttemptCount": 2, "refusals": [ { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0029498525073746312 } ], "rate": 0.0058997050147492625 }, "rewardHacking": null, "mean": { "reward": 0.684365781710915, "inputTokens": 15895334.887905605, "outputTokens": 406479.005899705 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 52, "hardStopAttemptCount": 0, "recoveredAttemptCount": 52, "fallbackAttemptCount": 52, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 25, "outcome": "fallback", "rate": 0.06720430107526881 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.005376344086021506 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 41, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 54, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 63, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 71, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 72, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 82, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 98, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 102, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 127, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 132, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 241, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 261, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 }, { "turn": 386, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.002688172043010753 } ], "rate": 0.13978494623655913 }, "rewardHacking": null, "mean": { "reward": 0.663978494623656, "inputTokens": 14842420.086021505, "outputTokens": 214788.37634408602 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 24, "hardStopAttemptCount": 0, "recoveredAttemptCount": 24, "fallbackAttemptCount": 24, "refusals": [ { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.015151515151515152 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.010101010101010102 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 44, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 58, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 88, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 126, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 360, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 436, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.005050505050505051 } ], "rate": 0.12121212121212122 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 125, "reviewedAttemptCount": 125, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 125 } ], "rate": 0 }, "mean": { "reward": 0.631313131313131, "inputTokens": 14764903.207070706, "outputTokens": 430101.28282828286 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 78, "hardStopAttemptCount": 0, "recoveredAttemptCount": 78, "fallbackAttemptCount": 78, "refusals": [ { "turn": 18, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 30, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0009832841691248771 }, { "turn": 0, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 25, "outcome": "fallback", "rate": 0.022401433691756272 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.001792114695340502 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 9, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 17, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 19, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 25, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 28, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 33, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 35, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 41, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 54, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 61, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 63, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 71, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 72, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 82, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 98, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 102, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 110, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 127, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 132, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 241, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 261, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 386, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.000896057347670251 }, { "turn": 0, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 3, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 3, "outcome": "fallback", "rate": 0.005050505050505051 }, { "turn": 5, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 2, "outcome": "fallback", "rate": 0.0033670033670033673 }, { "turn": 2, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 2, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 3, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 4, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 4, "chain": [ { "model": "claude-opus-5", "releaseSlug": "claude-opus-5", "releaseName": "Claude Opus 5", "releaseDate": "2026-07-24", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 6, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 21, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 44, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 45, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 48, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 58, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 88, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 126, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 258, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 360, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 }, { "turn": 436, "chain": [ { "model": "claude-opus-4-8", "releaseSlug": "claude-opus-4-8", "releaseName": "Claude Opus 4.8", "releaseDate": "2026-05-28", "creatorSlug": "anthropic" } ], "attemptCount": 1, "outcome": "fallback", "rate": 0.0016835016835016836 } ], "rate": 0.08896559082114319 }, "indexScore": 0.659885802549234, "mean": { "reward": 0.659885802549234, "costUsd": 13.036383418426828, "agentWallTimeSec": 3867.0923410341024, "steps": 155.0957095709571, "inputTokens": 15218206.418041805, "outputTokens": 333176.801980198, "cacheTokens": 14568923.550055025, "cacheHitRate": 0.9500106599458848, "totalTokens": 15551383.220022004 }, "sums": { "costUsd": 3950.02417578334 }, "percentiles": { "costUsd": { "p05": 4.819416574015403, "p25": 7.233748635478535, "p50": 11.484682617381733, "p75": 16.62301155423542, "p95": 27.619909799086923 }, "totalTokens": { "p05": 3916930.348074808, "p25": 7542033.144664456, "p50": 12988730.595159527, "p75": 20374948.502200224, "p95": 34511194.44961495 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "2.1.280", "dateReleased": null }, "max": { "version": "2.1.280", "dateReleased": null } } } }, { "id": "45493c2782f1bd94f6d30d4911f317a0", "isDefault": true, "isHighlighted": false, "agentName": "Opencode", "provider": "zai", "hostModelSlug": "zai_glm-5-3", "display": { "agent": "Opencode", "model": "GLM-5.3", "creator": { "agent": "Opencode", "model": "Z.ai" } }, "displayLabel": "Opencode - GLM-5.3 ({'reasoning_effort': 'max'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.613569321533923, "inputTokens": 16638607.92920354, "outputTokens": 75199.58702064896 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.594086021505376, "inputTokens": 6141052.228494624, "outputTokens": 28542.190860215054 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 80, "reviewedAttemptCount": 80, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 80 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.398989898989899, "inputTokens": 25175801.106060605, "outputTokens": 158541.98484848483 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.5355484140097326, "mean": { "reward": 0.5355484140097326, "costUsd": 4.241385833751375, "agentWallTimeSec": 2887.9556523652373, "steps": 107.04950495049505, "inputTokens": 14202165.166116614, "outputTokens": 74259.26072607259, "cacheTokens": 14007356.268426869, "cacheHitRate": 0.9745705628087887, "totalTokens": 14276424.426842693 }, "sums": { "costUsd": 1285.139907626667 }, "percentiles": { "costUsd": { "p05": 0.889673751515953, "p25": 1.8921473049724982, "p50": 2.9852793977557757, "p75": 5.713520085478545, "p95": 10.917436082209012 }, "totalTokens": { "p05": 2410964.3325632564, "p25": 5510060.859735972, "p50": 9868718.469746958, "p75": 19057305.483498357, "p95": 38946948.10616063 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "1.18.29", "dateReleased": "2026-09-04" }, "max": { "version": "1.18.29", "dateReleased": "2026-09-04" } } } }, { "id": "70a5d300021b9986f77d4198272ef1a5", "isDefault": false, "isHighlighted": false, "agentName": "Grok Build", "provider": "xai", "hostModelSlug": "xai_grok-4-6-xhigh", "display": { "agent": "Grok Build", "model": "Grok 4.6 (xhigh)", "creator": { "agent": "xAI", "model": "xAI" } }, "displayLabel": "Grok Build - Grok 4.6 (xhigh)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.648967551622419, "inputTokens": 6323126.3598820055, "outputTokens": 69583.08554572272 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 3, "hardStopAttemptCount": 3, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.005376344086021506 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.008064516129032258 }, "rewardHacking": null, "mean": { "reward": 0.583333333333333, "inputTokens": 2992096.1370967743, "outputTokens": 26161.698924731183 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 36, "reviewedAttemptCount": 36, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 36 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.176767676767677, "inputTokens": 8745920.277777778, "outputTokens": 88596.601010101 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 3, "hardStopAttemptCount": 3, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 1, "chain": [], "attemptCount": 2, "outcome": "hard_stop", "rate": 0.001792114695340502 }, { "turn": 2, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.0026881720430107525 }, "indexScore": 0.4696895205744764, "mean": { "reward": 0.4696895205744764, "costUsd": 3.5701432805280535, "agentWallTimeSec": 1167.3387480748086, "steps": 112.06160616061608, "inputTokens": 5487669.762376238, "outputTokens": 55954.83498349835, "cacheTokens": 5160616.836083612, "cacheHitRate": 0.9057792630165156, "totalTokens": 5543624.597359739 }, "sums": { "costUsd": 1081.753414000001 }, "percentiles": { "costUsd": { "p05": 1.175140282068208, "p25": 2.007830724972496, "p50": 2.783706352035202, "p75": 4.600172847084706, "p95": 7.456153753135318 }, "totalTokens": { "p05": 1341131.6409240938, "p25": 2646561.652915294, "p50": 4024738.3938393835, "p75": 7158372.716171614, "p95": 12765170.414081411 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } }, "swe-atlas-qna": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } }, "terminal-bench-v4": { "min": { "version": "1.0.13", "dateReleased": "2026-08-28" }, "max": { "version": "1.0.13", "dateReleased": "2026-08-28" } } } }, { "id": "b3c56ec69185d544992fd578edb75e8c", "isDefault": false, "isHighlighted": false, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_gpt-5-6-luna", "display": { "agent": "Codex", "model": "GPT-5.6 Luna (max)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-5.6 Luna (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.663716814159292, "inputTokens": 16195750.386430679, "outputTokens": 75781.00294985251 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.486559139784946, "inputTokens": 7864729.142473118, "outputTokens": 31795.564516129034 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 29, "reviewedAttemptCount": 29, "flaggedAttemptCount": 0, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 29 } ], "rate": 0 }, "mean": { "reward": 0.146464646464646, "inputTokens": 25363172.36868687, "outputTokens": 137870.4292929293 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4322468668029613, "mean": { "reward": 0.4322468668029613, "costUsd": 0.43802256393839384, "agentWallTimeSec": 1412.260883388339, "steps": 121.51375137513753, "inputTokens": 14783219.74807481, "outputTokens": 71304.79097909792, "cacheTokens": 14535765.897689763, "cacheHitRate": 0.9718803286808345, "totalTokens": 14854524.539053895 }, "sums": { "costUsd": 132.7208368733333 }, "percentiles": { "costUsd": { "p05": 0.11470550527832782, "p25": 0.19050873430693036, "p50": 0.2971945570627061, "p75": 0.5478214921672169, "p95": 1.2523768718877888 }, "totalTokens": { "p05": 2583111.566226621, "p25": 5468368.5495049525, "p50": 9575964.7370737, "p75": 19574390.483498316, "p95": 48106178.56754682 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } }, { "id": "542590ecc1aa12a718d158565d464daa", "isDefault": true, "isHighlighted": false, "agentName": "Codex", "provider": "deepseek", "hostModelSlug": "deepseek_deepseek-v4-pro-0813", "display": { "agent": "Codex", "model": "DeepSeek V4 Pro 0813 (max)", "creator": { "agent": "OpenAI", "model": "DeepSeek" } }, "displayLabel": "Codex - DeepSeek V4 Pro 0813 (max)", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.572271386430678, "inputTokens": 21857745.433628317, "outputTokens": 116676.62241887905 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.618279569892473, "inputTokens": 9901327.01075269, "outputTokens": 46364.06182795699 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 21, "reviewedAttemptCount": 21, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 21 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.101010101010101, "inputTokens": 54448314.87373737, "outputTokens": 247887.35606060605 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "indexScore": 0.4305203524444174, "mean": { "reward": 0.4305203524444174, "costUsd": 0.23804610720792058, "agentWallTimeSec": 2417.5500099009905, "steps": 138.14631463146316, "inputTokens": 24063625.627062704, "outputTokens": 116482.400990099, "cacheTokens": 23948932.43564355, "cacheHitRate": 0.9898485564269163, "totalTokens": 24180108.028052814 }, "sums": { "costUsd": 72.127970484 }, "percentiles": { "costUsd": { "p05": 0.08553589994158414, "p25": 0.13481908242684273, "p50": 0.20057701675687584, "p75": 0.30849491996424616, "p95": 0.4808581146649064 }, "totalTokens": { "p05": 3736359.9761276133, "p25": 8302592.635863575, "p50": 17833020.419141915, "p75": 32302640.28272825, "p95": 63649800.394829504 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "swe-atlas-qna": { "min": { "version": "0.147.0", "dateReleased": "2026-08-07" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } }, "terminal-bench-v4": { "min": { "version": "0.153.2", "dateReleased": "2026-09-03" }, "max": { "version": "0.153.4", "dateReleased": "2026-09-04" } } } }, { "id": "ed8c1d0b0a2fe4560bbfa0a9f47e8941", "isDefault": true, "isHighlighted": true, "agentName": "Codex", "provider": "openai", "hostModelSlug": "openai_quasar-alpha", "display": { "agent": "Codex", "model": "GPT-6.1 Sol (xhigh)", "creator": { "agent": "OpenAI", "model": "OpenAI" } }, "displayLabel": "Codex - GPT-6.1 Sol (xhigh) ({'reasoning_effort': 'xhigh'})", "variantOf": null, "isUnavailable": false, "indexComponentCount": 3, "evalCount": 3, "evals": [ { "datasetIndexName": "deep-swe-v1.1", "evaluationDatasetSlug": "DeepSWE v1.1", "refDatasetName": "datacurve/deep-swe-1-1@1", "weight": 0.333333333333333, "safety": { "attemptCount": 339, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": null, "mean": { "reward": 0.731563421828909, "inputTokens": 2690171.784660767, "outputTokens": 37762.082595870204 } }, { "datasetIndexName": "swe-atlas-qna", "evaluationDatasetSlug": "SWE-Atlas-QnA", "refDatasetName": "datasets/swe-atlas-qna", "weight": 0.333333333333333, "safety": { "attemptCount": 372, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.016129032258064516 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 15, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 18, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 20, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 }, { "turn": 27, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.002688172043010753 } ], "rate": 0.03494623655913978 }, "rewardHacking": null, "mean": { "reward": 0.610215053763441, "inputTokens": 2748674.931693989, "outputTokens": 21382.483606557376 } }, { "datasetIndexName": "terminal-bench-v4", "evaluationDatasetSlug": "Terminal-Bench v4", "refDatasetName": "terminal-bench/terminal-bench@sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", "weight": 0.333333333333333, "safety": { "attemptCount": 198, "refusedAttemptCount": 0, "hardStopAttemptCount": 0, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [], "rate": 0 }, "rewardHacking": { "retainedAttemptCount": 198, "passingAttemptCount": 109, "reviewedAttemptCount": 109, "flaggedAttemptCount": 1, "notApplicableAttemptCount": 0, "missingAuditAttemptCount": 0, "skippedAuditAttemptCount": 0, "unavailableAuditAttemptCount": 0, "gradingErrorAttemptCount": 0, "invalidAuditAttemptCount": 0, "snapshotMismatchAttemptCount": 0, "materializedAt": "2026-10-01T06:05:18.626423+00:00", "gradingVersion": "v1", "graderConfigHash": "e3e86c5c130946ce", "reviewSources": [ { "judgeModel": "sonnet", "rubricId": "ca-bench-reward-hacking-v1", "auditSchemaVersion": 1, "attemptCount": 109 } ], "rate": 0.005050505050505051 }, "mean": { "reward": 0.545454545454545, "inputTokens": 4815817, "outputTokens": 57703.767676767675 } } ], "safety": { "attemptCount": 909, "refusedAttemptCount": 13, "hardStopAttemptCount": 13, "recoveredAttemptCount": 0, "fallbackAttemptCount": 0, "refusals": [ { "turn": 0, "chain": [], "attemptCount": 6, "outcome": "hard_stop", "rate": 0.005376344086021505 }, { "turn": 3, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 11, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 13, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 15, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 18, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 20, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 }, { "turn": 27, "chain": [], "attemptCount": 1, "outcome": "hard_stop", "rate": 0.000896057347670251 } ], "rate": 0.01164874551971326 }, "indexScore": 0.6290776736822983, "mean": { "reward": 0.6290776736822983, "costUsd": 1.0397081014409635, "agentWallTimeSec": 931.3823091309132, "steps": 40.05500550055005, "inputTokens": 3177125.49569875, "outputTokens": 35402.61375317859, "cacheTokens": 3036065.628757956, "cacheHitRate": 0.9327572280083256, "totalTokens": 3212528.1094519314 }, "sums": { "costUsd": 313.4238835999993 }, "percentiles": { "costUsd": { "p05": 0.39666282002200215, "p25": 0.5709024621012102, "p50": 0.818101308030803, "p75": 1.1004568787128721, "p95": 2.1570592566556637 }, "totalTokens": { "p05": 682378.7798679869, "p25": 1314619.7282728287, "p50": 2203242.6336633656, "p75": 3691119.964796481, "p95": 7871107.464686463 } }, "versions": { "deep-swe-v1.1": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "swe-atlas-qna": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } }, "terminal-bench-v4": { "min": { "version": "0.154.0", "dateReleased": null }, "max": { "version": "0.154.0", "dateReleased": null } } } } ] }