{
 "schemaVersion": 1,
 "generatedAt": "2026-11-11T21:01:46Z",
 "repo": {
  "display": "click",
  "site": "https://github.com/pallets/click",
  "language": "python",
  "url": "https://github.com/pallets/click",
  "visibility": "public"
 },
 "method": {
  "oneTry": false,
  "reproduceFirst": false,
  "hiddenTests": true,
  "weightRule": null,
  "totalPoints": null,
  "trialsPerBug": 3
 },
 "models": [
  {
   "id": "claude-haiku-5-5",
   "display": "Claude Haiku 5.5",
   "effort": null,
   "vendor": "anthropic",
   "colour": "#ff8f66"
  },
  {
   "id": "gpt-6-luna",
   "display": "GPT-6 Luna",
   "effort": null,
   "vendor": "openai",
   "colour": "#5cc8ff"
  }
 ],
 "videos": [
  {
   "id": "click",
   "title": "click: 5 real bugs",
   "youtubeId": null,
   "publishedDate": "2026-11-11",
   "benchmarkId": "0e22574183bf",
   "contenders": [
    "claude-haiku-5-5",
    "gpt-6-luna"
   ],
   "bonus": [],
   "pricingBasis": null,
   "totals": {
    "claude-haiku-5-5": {
     "fixed": 5,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 581.3,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.8667,
       "low": 0.7333,
       "high": 1
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.8667
       },
       {
        "k": 2,
        "value": 1
       },
       {
        "k": 3,
        "value": 1
       }
      ],
      "firstTryRate": 0.2,
      "feedbackGain": 0.6667,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    },
    "gpt-6-luna": {
     "fixed": 3,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 387.1,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.6667,
       "low": 0.2667,
       "high": 1
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.6667
       },
       {
        "k": 2,
        "value": 0.7333
       },
       {
        "k": 3,
        "value": 0.8
       }
      ],
      "firstTryRate": 0.2,
      "feedbackGain": 0.4667,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    }
   }
  }
 ],
 "bugs": [
  {
   "scenarioId": "fb2423349962",
   "videoIds": [
    "click"
   ],
   "title": "Fix `show_envvar` output for a sequence or an empty `envvar`",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "bb695aaecdef95832d167fc1d508921745aa824e",
   "fixCommit": "c53d3732b98c764932e146c007146ad402acb229",
   "benchmarkId": "0e22574183bf",
   "links": {
    "dataset": {
     "task": "pallets__click__fb2423349962",
     "path": "harbor/pallets__click__fb2423349962"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/click/claude-haiku-5-5/fb2423349962.json",
      "replays/click/claude-haiku-5-5/fb2423349962.t2.json",
      "replays/click/claude-haiku-5-5/fb2423349962.t3.json"
     ],
     "gpt-6-luna": [
      "replays/click/gpt-6-luna/fb2423349962.json",
      "replays/click/gpt-6-luna/fb2423349962.t2.json",
      "replays/click/gpt-6-luna/fb2423349962.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "d80c3e643940",
   "videoIds": [
    "click"
   ],
   "title": "Fix short help abbreviation",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "6d5bac5c712195cbfa9666ba0ecbf0573cdea356",
   "fixCommit": "f67c2bb6c2f055c1c7a57cd46c3697fc26a7742e",
   "benchmarkId": "0e22574183bf",
   "links": {
    "dataset": {
     "task": "pallets__click__d80c3e643940",
     "path": "harbor/pallets__click__d80c3e643940"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/click/claude-haiku-5-5/d80c3e643940.json",
      "replays/click/claude-haiku-5-5/d80c3e643940.t2.json",
      "replays/click/claude-haiku-5-5/d80c3e643940.t3.json"
     ],
     "gpt-6-luna": [
      "replays/click/gpt-6-luna/d80c3e643940.json",
      "replays/click/gpt-6-luna/d80c3e643940.t2.json",
      "replays/click/gpt-6-luna/d80c3e643940.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "3bf57df9051a",
   "videoIds": [
    "click"
   ],
   "title": "Fix race condition on `KeyboardInterrupt` arriving in `Command.main()`",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "08a7bc6d2a2af02268b09b3b5fd8079546c81c03",
   "fixCommit": "fc518e41218b447e875fe27244ae692c269bcb77",
   "benchmarkId": "0e22574183bf",
   "links": {
    "dataset": {
     "task": "pallets__click__3bf57df9051a",
     "path": "harbor/pallets__click__3bf57df9051a"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/click/claude-haiku-5-5/3bf57df9051a.json",
      "replays/click/claude-haiku-5-5/3bf57df9051a.t2.json",
      "replays/click/claude-haiku-5-5/3bf57df9051a.t3.json"
     ],
     "gpt-6-luna": [
      "replays/click/gpt-6-luna/3bf57df9051a.json",
      "replays/click/gpt-6-luna/3bf57df9051a.t2.json",
      "replays/click/gpt-6-luna/3bf57df9051a.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "10c2182297ef",
   "videoIds": [
    "click"
   ],
   "title": "Fix `copy`, `deepcopy` and `pickle` of `Sentinel` members",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "420c8fb44eeadb537cae69d2fee3796e808558dd",
   "fixCommit": "f58ca3e81424a35626c8a475eb59ab95589008ce",
   "benchmarkId": "0e22574183bf",
   "links": {
    "dataset": {
     "task": "pallets__click__10c2182297ef",
     "path": "harbor/pallets__click__10c2182297ef"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/click/claude-haiku-5-5/10c2182297ef.json",
      "replays/click/claude-haiku-5-5/10c2182297ef.t2.json",
      "replays/click/claude-haiku-5-5/10c2182297ef.t3.json"
     ],
     "gpt-6-luna": [
      "replays/click/gpt-6-luna/10c2182297ef.json",
      "replays/click/gpt-6-luna/10c2182297ef.t2.json",
      "replays/click/gpt-6-luna/10c2182297ef.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "bb50b9f8ffa4",
   "videoIds": [
    "click"
   ],
   "title": "Forwards all user's parameters set in `PAGER`",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "61b69e967e525bd502f5bf42def4d551e761fe0e",
   "fixCommit": "a6256bfb5971d5e58585fe7b6c656134e1ade5a4",
   "benchmarkId": "0e22574183bf",
   "links": {
    "dataset": {
     "task": "pallets__click__bb50b9f8ffa4",
     "path": "harbor/pallets__click__bb50b9f8ffa4"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/click/claude-haiku-5-5/bb50b9f8ffa4.json",
      "replays/click/claude-haiku-5-5/bb50b9f8ffa4.t2.json",
      "replays/click/claude-haiku-5-5/bb50b9f8ffa4.t3.json"
     ],
     "gpt-6-luna": [
      "replays/click/gpt-6-luna/bb50b9f8ffa4.json",
      "replays/click/gpt-6-luna/bb50b9f8ffa4.t2.json",
      "replays/click/gpt-6-luna/bb50b9f8ffa4.t3.json"
     ]
    }
   }
  }
 ],
 "runs": [
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "fb2423349962",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 82.786,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/fb2423349962.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "b13135e23830"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "d80c3e643940",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 249.412,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/d80c3e643940.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "b13135e23830"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "3bf57df9051a",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 173.321,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/3bf57df9051a.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "b13135e23830"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "10c2182297ef",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 71.009,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/10c2182297ef.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "b13135e23830"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "bb50b9f8ffa4",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 136.186,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/bb50b9f8ffa4.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "b13135e23830"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "fb2423349962",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 103.268,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/fb2423349962.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2a6a73ec4da8"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "d80c3e643940",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 200.137,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/d80c3e643940.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2a6a73ec4da8"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "3bf57df9051a",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 134.656,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/3bf57df9051a.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2a6a73ec4da8"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "10c2182297ef",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 34.621,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/10c2182297ef.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2a6a73ec4da8"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "bb50b9f8ffa4",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 115.561,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/bb50b9f8ffa4.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2a6a73ec4da8"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "fb2423349962",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 61.784,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/fb2423349962.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "79a9777c4f66"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "d80c3e643940",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 153.59,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/d80c3e643940.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "79a9777c4f66"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "3bf57df9051a",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 80.506,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/3bf57df9051a.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "79a9777c4f66"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "10c2182297ef",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 37.268,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/10c2182297ef.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "79a9777c4f66"
  },
  {
   "videoId": "click",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "bb50b9f8ffa4",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 109.928,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/claude-haiku-5-5/bb50b9f8ffa4.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ae115d46f0c8",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "79a9777c4f66"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "fb2423349962",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 84.515,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/fb2423349962.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "5858cef114b4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "d80c3e643940",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 91.45,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/d80c3e643940.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "5858cef114b4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "3bf57df9051a",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 62.349,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/3bf57df9051a.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "5858cef114b4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "10c2182297ef",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 60.869,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/10c2182297ef.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "5858cef114b4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "bb50b9f8ffa4",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 105.989,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/bb50b9f8ffa4.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "5858cef114b4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "fb2423349962",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 81.63,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/fb2423349962.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "70ceb40cfeb1"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "d80c3e643940",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 78.453,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/d80c3e643940.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "70ceb40cfeb1"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "3bf57df9051a",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 75.304,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/3bf57df9051a.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "70ceb40cfeb1"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "10c2182297ef",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 63.66,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/10c2182297ef.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "70ceb40cfeb1"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "bb50b9f8ffa4",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 103.951,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/bb50b9f8ffa4.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "70ceb40cfeb1"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "fb2423349962",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 95.221,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/fb2423349962.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "e8449d892fd4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "d80c3e643940",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 75.085,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/d80c3e643940.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "e8449d892fd4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "3bf57df9051a",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 70.977,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/3bf57df9051a.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "e8449d892fd4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "10c2182297ef",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 29.011,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/10c2182297ef.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "e8449d892fd4"
  },
  {
   "videoId": "click",
   "modelId": "gpt-6-luna",
   "scenarioId": "bb50b9f8ffa4",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 82.806,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/click/gpt-6-luna/bb50b9f8ffa4.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "5b3dff9e4d5e",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "e8449d892fd4"
  }
 ],
 "leaderboard": {
  "basis": "5 bugs, every trial counts; fixed = bugs resolved in at least half their trials",
  "bugCount": 5,
  "maxPoints": null,
  "omittedBugCount": 0,
  "rows": [
   {
    "modelId": "claude-haiku-5-5",
    "videoId": "click",
    "fixed": 5,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 581.3,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.8667,
      "low": 0.7333,
      "high": 1
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.8667
      },
      {
       "k": 2,
       "value": 1
      },
      {
       "k": 3,
       "value": 1
      }
     ],
     "firstTryRate": 0.2,
     "feedbackGain": 0.6667,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "ae115d46f0c8"
   },
   {
    "modelId": "gpt-6-luna",
    "videoId": "click",
    "fixed": 3,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 387.1,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.6667,
      "low": 0.2667,
      "high": 1
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.6667
      },
      {
       "k": 2,
       "value": 0.7333
      },
      {
       "k": 3,
       "value": 0.8
      }
     ],
     "firstTryRate": 0.2,
     "feedbackGain": 0.4667,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "5b3dff9e4d5e"
   }
  ]
 }
}