{ "schemaVersion": 1, "generatedAt": "2026-11-11T21:02:12Z", "repo": { "display": "toml", "site": "https://github.com/toml-rs/toml", "language": "rust", "url": "https://github.com/toml-rs/toml", "visibility": "public" }, "method": { "oneTry": false, "reproduceFirst": false, "hiddenTests": true, "weightRule": null, "totalPoints": null, "trialsPerBug": 3 }, "models": [ { "id": "claude-haiku-5-5", "display": "Claude Haiku 5.5", "effort": null, "vendor": "anthropic", "colour": "#ff8f66" }, { "id": "gpt-6-luna", "display": "GPT-6 Luna", "effort": null, "vendor": "openai", "colour": "#5cc8ff" } ], "videos": [ { "id": "toml", "title": "toml: 5 real bugs", "youtubeId": null, "publishedDate": "2026-11-11", "benchmarkId": "321567dbfd20", "contenders": [ "claude-haiku-5-5", "gpt-6-luna" ], "bonus": [], "pricingBasis": null, "totals": { "claude-haiku-5-5": { "fixed": 4, "attempted": 5, "points": null, "maxPoints": null, "costUSD": null, "timeSeconds": 594.8, "tokens": null, "statistics": { "bugCount": 5, "trialsPerBug": 3, "trialCount": 15, "resolveRate": { "mean": 0.8, "low": 0.4, "high": 1 }, "passAtK": [ { "k": 1, "value": 0.8 }, { "k": 2, "value": 0.8 }, { "k": 3, "value": 0.8 } ], "firstTryRate": 0.8, "feedbackGain": 0, "confidence": 0.95, "bootstrapSamples": 2000, "seed": 166 } }, "gpt-6-luna": { "fixed": 4, "attempted": 5, "points": null, "maxPoints": null, "costUSD": null, "timeSeconds": 454.4, "tokens": null, "statistics": { "bugCount": 5, "trialsPerBug": 3, "trialCount": 15, "resolveRate": { "mean": 0.7333, "low": 0.3333, "high": 1 }, "passAtK": [ { "k": 1, "value": 0.7333 }, { "k": 2, "value": 0.8 }, { "k": 3, "value": 0.8 } ], "firstTryRate": 0.6, "feedbackGain": 0.1333, "confidence": 0.95, "bootstrapSamples": 2000, "seed": 166 } } } } ], "bugs": [ { "scenarioId": "8775f471a800", "videoIds": [ "toml" ], "title": "fix(display): Ensure all keys are encoded", "plainTitle": null, "explanation": null, "points": null, "linesChanged": null, "inCommonBoard": true, "buggyCommit": "711b1ba1ecb11e61df9a1609e24fa3b17d11d961", "fixCommit": "76484d005a0e68e8c1876f8a3417b71a3921623f", "benchmarkId": "321567dbfd20", "links": { "dataset": { "task": "toml-rs__toml__8775f471a800", "path": "harbor/toml-rs__toml__8775f471a800" }, "video": null, "replays": { "claude-haiku-5-5": [ "replays/toml/claude-haiku-5-5/8775f471a800.json", "replays/toml/claude-haiku-5-5/8775f471a800.t2.json", "replays/toml/claude-haiku-5-5/8775f471a800.t3.json" ], "gpt-6-luna": [ "replays/toml/gpt-6-luna/8775f471a800.json", "replays/toml/gpt-6-luna/8775f471a800.t2.json", "replays/toml/gpt-6-luna/8775f471a800.t3.json" ] } } }, { "scenarioId": "ac6aef993028", "videoIds": [ "toml" ], "title": "fix(serde): Deserialize Value datetimes into typed targets", "plainTitle": null, "explanation": null, "points": null, "linesChanged": null, "inCommonBoard": true, "buggyCommit": "6e45cef5d5061fee00fe43cb6b52c3126e05d57a", "fixCommit": "534039ccff11412422b3d365d092ef2ed43df2ac", "benchmarkId": "321567dbfd20", "links": { "dataset": { "task": "toml-rs__toml__ac6aef993028", "path": "harbor/toml-rs__toml__ac6aef993028" }, "video": null, "replays": { "claude-haiku-5-5": [ "replays/toml/claude-haiku-5-5/ac6aef993028.json", "replays/toml/claude-haiku-5-5/ac6aef993028.t2.json", "replays/toml/claude-haiku-5-5/ac6aef993028.t3.json" ], "gpt-6-luna": [ "replays/toml/gpt-6-luna/ac6aef993028.json", "replays/toml/gpt-6-luna/ac6aef993028.t2.json", "replays/toml/gpt-6-luna/ac6aef993028.t3.json" ] } } }, { "scenarioId": "6c0a2aff6191", "videoIds": [ "toml" ], "title": "fix(writer): Don't overflow", "plainTitle": null, "explanation": null, "points": null, "linesChanged": null, "inCommonBoard": true, "buggyCommit": "5e70a7031d7cd667bdc081419373df4b74bf78d8", "fixCommit": "fb0c1b376d291ee4ffdf5297a67579f2ad7ce70d", "benchmarkId": "321567dbfd20", "links": { "dataset": { "task": "toml-rs__toml__6c0a2aff6191", "path": "harbor/toml-rs__toml__6c0a2aff6191" }, "video": null, "replays": { "claude-haiku-5-5": [ "replays/toml/claude-haiku-5-5/6c0a2aff6191.json", "replays/toml/claude-haiku-5-5/6c0a2aff6191.t2.json", "replays/toml/claude-haiku-5-5/6c0a2aff6191.t3.json" ], "gpt-6-luna": [ "replays/toml/gpt-6-luna/6c0a2aff6191.json", "replays/toml/gpt-6-luna/6c0a2aff6191.t2.json", "replays/toml/gpt-6-luna/6c0a2aff6191.t3.json" ] } } }, { "scenarioId": "bded46009ca0", "videoIds": [ "toml" ], "title": "fix(edit): Don't allow moving tables above root", "plainTitle": null, "explanation": null, "points": null, "linesChanged": null, "inCommonBoard": true, "buggyCommit": "a2d4bf492ceed2b4eb255ffde78603a82404e3de", "fixCommit": "05fe0ce66848545a4131081158e3e32a5351365e", "benchmarkId": "321567dbfd20", "links": { "dataset": { "task": "toml-rs__toml__bded46009ca0", "path": "harbor/toml-rs__toml__bded46009ca0" }, "video": null, "replays": { "claude-haiku-5-5": [ "replays/toml/claude-haiku-5-5/bded46009ca0.json", "replays/toml/claude-haiku-5-5/bded46009ca0.t2.json", "replays/toml/claude-haiku-5-5/bded46009ca0.t3.json" ], "gpt-6-luna": [ "replays/toml/gpt-6-luna/bded46009ca0.json", "replays/toml/gpt-6-luna/bded46009ca0.t2.json", "replays/toml/gpt-6-luna/bded46009ca0.t3.json" ] } } }, { "scenarioId": "0ef14ecced23", "videoIds": [ "toml" ], "title": "fix(toml): Track dotted key for spans", "plainTitle": null, "explanation": null, "points": null, "linesChanged": null, "inCommonBoard": true, "buggyCommit": "796370e7d2d5d263384515d8b76a1e41e9c74cf6", "fixCommit": "87461d819f3383c26b8d9c4e70db208075a7a0d8", "benchmarkId": "321567dbfd20", "links": { "dataset": { "task": "toml-rs__toml__0ef14ecced23", "path": "harbor/toml-rs__toml__0ef14ecced23" }, "video": null, "replays": { "claude-haiku-5-5": [ "replays/toml/claude-haiku-5-5/0ef14ecced23.json", "replays/toml/claude-haiku-5-5/0ef14ecced23.t2.json", "replays/toml/claude-haiku-5-5/0ef14ecced23.t3.json" ], "gpt-6-luna": [ "replays/toml/gpt-6-luna/0ef14ecced23.json", "replays/toml/gpt-6-luna/0ef14ecced23.t2.json", "replays/toml/gpt-6-luna/0ef14ecced23.t3.json" ] } } } ], "runs": [ { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "8775f471a800", "passed": true, "reproducedFirst": null, "durationSeconds": 205.399, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/8775f471a800.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 1, "trialCount": 3, "runId": "ba3a96dcf1bb" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "ac6aef993028", "passed": true, "reproducedFirst": null, "durationSeconds": 156.904, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/ac6aef993028.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 1, "trialCount": 3, "runId": "ba3a96dcf1bb" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "6c0a2aff6191", "passed": true, "reproducedFirst": null, "durationSeconds": 77.835, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/6c0a2aff6191.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 1, "trialCount": 3, "runId": "ba3a96dcf1bb" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "bded46009ca0", "passed": true, "reproducedFirst": null, "durationSeconds": 78.967, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/bded46009ca0.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 1, "trialCount": 3, "runId": "ba3a96dcf1bb" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "0ef14ecced23", "passed": false, "reproducedFirst": null, "durationSeconds": 234.598, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/0ef14ecced23.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 1, "trialCount": 3, "runId": "ba3a96dcf1bb" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "8775f471a800", "passed": true, "reproducedFirst": null, "durationSeconds": 125.428, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/8775f471a800.t2.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 2, "trialCount": 3, "runId": "70aa9bc54285" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "ac6aef993028", "passed": true, "reproducedFirst": null, "durationSeconds": 131.421, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/ac6aef993028.t2.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 2, "trialCount": 3, "runId": "70aa9bc54285" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "6c0a2aff6191", "passed": true, "reproducedFirst": null, "durationSeconds": 55.321, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/6c0a2aff6191.t2.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 2, "trialCount": 3, "runId": "70aa9bc54285" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "bded46009ca0", "passed": true, "reproducedFirst": null, "durationSeconds": 46.781, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/bded46009ca0.t2.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 2, "trialCount": 3, "runId": "70aa9bc54285" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "0ef14ecced23", "passed": false, "reproducedFirst": null, "durationSeconds": 179.568, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/0ef14ecced23.t2.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 2, "trialCount": 3, "runId": "70aa9bc54285" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "8775f471a800", "passed": true, "reproducedFirst": null, "durationSeconds": 90.856, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/8775f471a800.t3.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 3, "trialCount": 3, "runId": "8a127fa47590" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "ac6aef993028", "passed": true, "reproducedFirst": null, "durationSeconds": 96.452, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/ac6aef993028.t3.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 3, "trialCount": 3, "runId": "8a127fa47590" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "6c0a2aff6191", "passed": true, "reproducedFirst": null, "durationSeconds": 49.14, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/6c0a2aff6191.t3.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 3, "trialCount": 3, "runId": "8a127fa47590" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "bded46009ca0", "passed": true, "reproducedFirst": null, "durationSeconds": 55.301, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/bded46009ca0.t3.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 3, "trialCount": 3, "runId": "8a127fa47590" }, { "videoId": "toml", "modelId": "claude-haiku-5-5", "scenarioId": "0ef14ecced23", "passed": false, "reproducedFirst": null, "durationSeconds": 200.414, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/claude-haiku-5-5/0ef14ecced23.t3.json", "agentCli": "claude-code", "agentCliSource": "RunDocument.agent", "runSetId": "d4a34de7bd5d", "trialIndex": 3, "trialCount": 3, "runId": "8a127fa47590" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "8775f471a800", "passed": true, "reproducedFirst": null, "durationSeconds": 141.46, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/8775f471a800.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 1, "trialCount": 3, "runId": "631c977ebc21" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "ac6aef993028", "passed": true, "reproducedFirst": null, "durationSeconds": 107.471, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/ac6aef993028.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 1, "trialCount": 3, "runId": "631c977ebc21" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "6c0a2aff6191", "passed": true, "reproducedFirst": null, "durationSeconds": 64.164, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/6c0a2aff6191.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 1, "trialCount": 3, "runId": "631c977ebc21" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "bded46009ca0", "passed": true, "reproducedFirst": null, "durationSeconds": 76.747, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/bded46009ca0.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 1, "trialCount": 3, "runId": "631c977ebc21" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "0ef14ecced23", "passed": false, "reproducedFirst": null, "durationSeconds": 157.63, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/0ef14ecced23.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 1, "trialCount": 3, "runId": "631c977ebc21" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "8775f471a800", "passed": true, "reproducedFirst": null, "durationSeconds": 102.222, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/8775f471a800.t2.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 2, "trialCount": 3, "runId": "99deab96809e" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "ac6aef993028", "passed": true, "reproducedFirst": null, "durationSeconds": 127.077, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/ac6aef993028.t2.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 2, "trialCount": 3, "runId": "99deab96809e" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "6c0a2aff6191", "passed": false, "reproducedFirst": null, "durationSeconds": 49.851, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/6c0a2aff6191.t2.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 2, "trialCount": 3, "runId": "99deab96809e" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "bded46009ca0", "passed": true, "reproducedFirst": null, "durationSeconds": 73.865, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/bded46009ca0.t2.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 2, "trialCount": 3, "runId": "99deab96809e" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "0ef14ecced23", "passed": false, "reproducedFirst": null, "durationSeconds": 109.162, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/0ef14ecced23.t2.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 2, "trialCount": 3, "runId": "99deab96809e" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "8775f471a800", "passed": true, "reproducedFirst": null, "durationSeconds": 49.469, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/8775f471a800.t3.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 3, "trialCount": 3, "runId": "5d866a7a2a90" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "ac6aef993028", "passed": true, "reproducedFirst": null, "durationSeconds": 95.855, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/ac6aef993028.t3.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 3, "trialCount": 3, "runId": "5d866a7a2a90" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "6c0a2aff6191", "passed": true, "reproducedFirst": null, "durationSeconds": 60.045, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/6c0a2aff6191.t3.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 3, "trialCount": 3, "runId": "5d866a7a2a90" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "bded46009ca0", "passed": true, "reproducedFirst": null, "durationSeconds": 41.305, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/bded46009ca0.t3.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 3, "trialCount": 3, "runId": "5d866a7a2a90" }, { "videoId": "toml", "modelId": "gpt-6-luna", "scenarioId": "0ef14ecced23", "passed": false, "reproducedFirst": null, "durationSeconds": 106.792, "tokens": null, "costUSD": null, "timedOut": false, "blindOverall": null, "replay": "replays/toml/gpt-6-luna/0ef14ecced23.t3.json", "agentCli": "codex", "agentCliSource": "RunDocument.agent", "runSetId": "e6d3d3ddf9ea", "trialIndex": 3, "trialCount": 3, "runId": "5d866a7a2a90" } ], "leaderboard": { "basis": "5 bugs, every trial counts; fixed = bugs resolved in at least half their trials", "bugCount": 5, "maxPoints": null, "omittedBugCount": 0, "rows": [ { "modelId": "claude-haiku-5-5", "videoId": "toml", "fixed": 4, "of": 5, "points": null, "costUSD": null, "timeSeconds": 594.8, "tokens": null, "runCount": 5, "statistics": { "bugCount": 5, "trialsPerBug": 3, "trialCount": 15, "resolveRate": { "mean": 0.8, "low": 0.4, "high": 1 }, "passAtK": [ { "k": 1, "value": 0.8 }, { "k": 2, "value": 0.8 }, { "k": 3, "value": 0.8 } ], "firstTryRate": 0.8, "feedbackGain": 0, "confidence": 0.95, "bootstrapSamples": 2000, "seed": 166 }, "trials": 3, "runSetId": "d4a34de7bd5d" }, { "modelId": "gpt-6-luna", "videoId": "toml", "fixed": 4, "of": 5, "points": null, "costUSD": null, "timeSeconds": 454.4, "tokens": null, "runCount": 5, "statistics": { "bugCount": 5, "trialsPerBug": 3, "trialCount": 15, "resolveRate": { "mean": 0.7333, "low": 0.3333, "high": 1 }, "passAtK": [ { "k": 1, "value": 0.7333 }, { "k": 2, "value": 0.8 }, { "k": 3, "value": 0.8 } ], "firstTryRate": 0.6, "feedbackGain": 0.1333, "confidence": 0.95, "bootstrapSamples": 2000, "seed": 166 }, "trials": 3, "runSetId": "e6d3d3ddf9ea" } ] } }