{
 "schemaVersion": 1,
 "generatedAt": "2026-11-11T21:02:02Z",
 "repo": {
  "display": "jsoup",
  "site": "https://github.com/jhy/jsoup",
  "language": "java",
  "url": "https://github.com/jhy/jsoup",
  "visibility": "public"
 },
 "method": {
  "oneTry": false,
  "reproduceFirst": false,
  "hiddenTests": true,
  "weightRule": null,
  "totalPoints": null,
  "trialsPerBug": 3
 },
 "models": [
  {
   "id": "claude-haiku-5-5",
   "display": "Claude Haiku 5.5",
   "effort": null,
   "vendor": "anthropic",
   "colour": "#ff8f66"
  },
  {
   "id": "gpt-6-luna",
   "display": "GPT-6 Luna",
   "effort": null,
   "vendor": "openai",
   "colour": "#5cc8ff"
  }
 ],
 "videos": [
  {
   "id": "jsoup",
   "title": "jsoup: 5 real bugs",
   "youtubeId": null,
   "publishedDate": "2026-11-11",
   "benchmarkId": "5cfa40664a01",
   "contenders": [
    "claude-haiku-5-5",
    "gpt-6-luna"
   ],
   "bonus": [],
   "pricingBasis": null,
   "totals": {
    "claude-haiku-5-5": {
     "fixed": 3,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 1154.2,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.6,
       "low": 0.2667,
       "high": 0.9333
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.6
       },
       {
        "k": 2,
        "value": 0.7333
       },
       {
        "k": 3,
        "value": 0.8
       }
      ],
      "firstTryRate": 0.1333,
      "feedbackGain": 0.4667,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    },
    "gpt-6-luna": {
     "fixed": 3,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 603.1,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.4667,
       "low": 0.1333,
       "high": 0.8
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.4667
       },
       {
        "k": 2,
        "value": 0.6
       },
       {
        "k": 3,
        "value": 0.6
       }
      ],
      "firstTryRate": 0.2,
      "feedbackGain": 0.2667,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    }
   }
  }
 ],
 "bugs": [
  {
   "scenarioId": "871dac3d4a74",
   "videoIds": [
    "jsoup"
   ],
   "title": "Add `RcDataRecover` tag option",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "659b7d27d32a9bbe082bcc19224693bc1e957b4d",
   "fixCommit": "d90d57521d931db7bf7ee464a4c0a1a9293db088",
   "benchmarkId": "5cfa40664a01",
   "links": {
    "dataset": {
     "task": "jhy__jsoup__871dac3d4a74",
     "path": "harbor/jhy__jsoup__871dac3d4a74"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/jsoup/claude-haiku-5-5/871dac3d4a74.json",
      "replays/jsoup/claude-haiku-5-5/871dac3d4a74.t2.json",
      "replays/jsoup/claude-haiku-5-5/871dac3d4a74.t3.json"
     ],
     "gpt-6-luna": [
      "replays/jsoup/gpt-6-luna/871dac3d4a74.json",
      "replays/jsoup/gpt-6-luna/871dac3d4a74.t2.json",
      "replays/jsoup/gpt-6-luna/871dac3d4a74.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "fe2fa19e35d5",
   "videoIds": [
    "jsoup"
   ],
   "title": "Exclude cached children from attribute clone",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "b85c4ab23020632871216a3be9925e99bdead540",
   "fixCommit": "659b7d27d32a9bbe082bcc19224693bc1e957b4d",
   "benchmarkId": "5cfa40664a01",
   "links": {
    "dataset": {
     "task": "jhy__jsoup__fe2fa19e35d5",
     "path": "harbor/jhy__jsoup__fe2fa19e35d5"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/jsoup/claude-haiku-5-5/fe2fa19e35d5.json",
      "replays/jsoup/claude-haiku-5-5/fe2fa19e35d5.t2.json",
      "replays/jsoup/claude-haiku-5-5/fe2fa19e35d5.t3.json"
     ],
     "gpt-6-luna": [
      "replays/jsoup/gpt-6-luna/fe2fa19e35d5.json",
      "replays/jsoup/gpt-6-luna/fe2fa19e35d5.t2.json",
      "replays/jsoup/gpt-6-luna/fe2fa19e35d5.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "1a370e688289",
   "videoIds": [
    "jsoup"
   ],
   "title": "Clean up comment and incomplete tag tokenisation (#2636)",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "1aa1234ef75311db6e0fc73504d2815d9b67b4ff",
   "fixCommit": "0dc96b3b144c2eae59bcb1d5e1b412269ee50f2b",
   "benchmarkId": "5cfa40664a01",
   "links": {
    "dataset": {
     "task": "jhy__jsoup__1a370e688289",
     "path": "harbor/jhy__jsoup__1a370e688289"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/jsoup/claude-haiku-5-5/1a370e688289.json",
      "replays/jsoup/claude-haiku-5-5/1a370e688289.t2.json",
      "replays/jsoup/claude-haiku-5-5/1a370e688289.t3.json"
     ],
     "gpt-6-luna": [
      "replays/jsoup/gpt-6-luna/1a370e688289.json",
      "replays/jsoup/gpt-6-luna/1a370e688289.t2.json",
      "replays/jsoup/gpt-6-luna/1a370e688289.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "c8ce6dca41a8",
   "videoIds": [
    "jsoup"
   ],
   "title": "Correct fragment ignores",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "118acb619ea8ad6a8dbcd829edda9d4e1c0f3dcf",
   "fixCommit": "f062a8abb0271bf3a98a698f9b74fb2ac8f0c1e3",
   "benchmarkId": "5cfa40664a01",
   "links": {
    "dataset": {
     "task": "jhy__jsoup__c8ce6dca41a8",
     "path": "harbor/jhy__jsoup__c8ce6dca41a8"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/jsoup/claude-haiku-5-5/c8ce6dca41a8.json",
      "replays/jsoup/claude-haiku-5-5/c8ce6dca41a8.t2.json",
      "replays/jsoup/claude-haiku-5-5/c8ce6dca41a8.t3.json"
     ],
     "gpt-6-luna": [
      "replays/jsoup/gpt-6-luna/c8ce6dca41a8.json",
      "replays/jsoup/gpt-6-luna/c8ce6dca41a8.t2.json",
      "replays/jsoup/gpt-6-luna/c8ce6dca41a8.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "39b6f919ef26",
   "videoIds": [
    "jsoup"
   ],
   "title": "Fix attribute equals and hashcode to exclude internal (transient) data attributes",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "ea6d5bae43ada574e650814690ffa3622416daa6",
   "fixCommit": "118acb619ea8ad6a8dbcd829edda9d4e1c0f3dcf",
   "benchmarkId": "5cfa40664a01",
   "links": {
    "dataset": {
     "task": "jhy__jsoup__39b6f919ef26",
     "path": "harbor/jhy__jsoup__39b6f919ef26"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/jsoup/claude-haiku-5-5/39b6f919ef26.json",
      "replays/jsoup/claude-haiku-5-5/39b6f919ef26.t2.json",
      "replays/jsoup/claude-haiku-5-5/39b6f919ef26.t3.json"
     ],
     "gpt-6-luna": [
      "replays/jsoup/gpt-6-luna/39b6f919ef26.json",
      "replays/jsoup/gpt-6-luna/39b6f919ef26.t2.json",
      "replays/jsoup/gpt-6-luna/39b6f919ef26.t3.json"
     ]
    }
   }
  }
 ],
 "runs": [
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "871dac3d4a74",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 315.373,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/871dac3d4a74.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "690239ecb1f0"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "fe2fa19e35d5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 322.033,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/fe2fa19e35d5.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "690239ecb1f0"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "1a370e688289",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 405.915,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/1a370e688289.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "690239ecb1f0"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "c8ce6dca41a8",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 533.199,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/c8ce6dca41a8.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "690239ecb1f0"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "39b6f919ef26",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 141.107,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/39b6f919ef26.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "690239ecb1f0"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "871dac3d4a74",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 331.284,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/871dac3d4a74.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "f83fc8e063c6"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "fe2fa19e35d5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 102.146,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/fe2fa19e35d5.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "f83fc8e063c6"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "1a370e688289",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 279.117,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/1a370e688289.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "f83fc8e063c6"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "c8ce6dca41a8",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 110.983,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/c8ce6dca41a8.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "f83fc8e063c6"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "39b6f919ef26",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 104.408,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/39b6f919ef26.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "f83fc8e063c6"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "871dac3d4a74",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 318.872,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/871dac3d4a74.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "5bb865faa9ce"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "fe2fa19e35d5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 146.057,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/fe2fa19e35d5.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "5bb865faa9ce"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "1a370e688289",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 148.31,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/1a370e688289.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "5bb865faa9ce"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "c8ce6dca41a8",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 104.349,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/c8ce6dca41a8.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "5bb865faa9ce"
  },
  {
   "videoId": "jsoup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "39b6f919ef26",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 99.477,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/claude-haiku-5-5/39b6f919ef26.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "28beb12cbc40",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "5bb865faa9ce"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "871dac3d4a74",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 173.496,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/871dac3d4a74.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "8269bb939ee2"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "fe2fa19e35d5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 63.153,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/fe2fa19e35d5.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "8269bb939ee2"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "1a370e688289",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 221.327,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/1a370e688289.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "8269bb939ee2"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "c8ce6dca41a8",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 84.813,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/c8ce6dca41a8.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "8269bb939ee2"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "39b6f919ef26",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 164.699,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/39b6f919ef26.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "8269bb939ee2"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "871dac3d4a74",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 196.988,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/871dac3d4a74.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "b35d80f5efc7"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "fe2fa19e35d5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 66.119,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/fe2fa19e35d5.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "b35d80f5efc7"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "1a370e688289",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 106.464,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/1a370e688289.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "b35d80f5efc7"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "c8ce6dca41a8",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 106.821,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/c8ce6dca41a8.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "b35d80f5efc7"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "39b6f919ef26",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 80.16,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/39b6f919ef26.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "b35d80f5efc7"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "871dac3d4a74",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 107.886,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/871dac3d4a74.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "df601847182a"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "fe2fa19e35d5",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 100.806,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/fe2fa19e35d5.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "df601847182a"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "1a370e688289",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 147.713,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/1a370e688289.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "df601847182a"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "c8ce6dca41a8",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 77.381,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/c8ce6dca41a8.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "df601847182a"
  },
  {
   "videoId": "jsoup",
   "modelId": "gpt-6-luna",
   "scenarioId": "39b6f919ef26",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 111.571,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/jsoup/gpt-6-luna/39b6f919ef26.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "d7734d825b5a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "df601847182a"
  }
 ],
 "leaderboard": {
  "basis": "5 bugs, every trial counts; fixed = bugs resolved in at least half their trials",
  "bugCount": 5,
  "maxPoints": null,
  "omittedBugCount": 0,
  "rows": [
   {
    "modelId": "claude-haiku-5-5",
    "videoId": "jsoup",
    "fixed": 3,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 1154.2,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.6,
      "low": 0.2667,
      "high": 0.9333
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.6
      },
      {
       "k": 2,
       "value": 0.7333
      },
      {
       "k": 3,
       "value": 0.8
      }
     ],
     "firstTryRate": 0.1333,
     "feedbackGain": 0.4667,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "28beb12cbc40"
   },
   {
    "modelId": "gpt-6-luna",
    "videoId": "jsoup",
    "fixed": 3,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 603.1,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.4667,
      "low": 0.1333,
      "high": 0.8
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.4667
      },
      {
       "k": 2,
       "value": 0.6
      },
      {
       "k": 3,
       "value": 0.6
      }
     ],
     "firstTryRate": 0.2,
     "feedbackGain": 0.2667,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "d7734d825b5a"
   }
  ]
 }
}