{
 "schemaVersion": 1,
 "generatedAt": "2026-11-11T21:18:58Z",
 "repo": {
  "display": "yup",
  "site": "https://github.com/jquense/yup",
  "language": "typescript",
  "url": "https://github.com/jquense/yup",
  "visibility": "public"
 },
 "method": {
  "oneTry": false,
  "reproduceFirst": false,
  "hiddenTests": true,
  "weightRule": null,
  "totalPoints": null,
  "trialsPerBug": 3
 },
 "models": [
  {
   "id": "claude-haiku-5-5",
   "display": "Claude Haiku 5.5",
   "effort": null,
   "vendor": "anthropic",
   "colour": "#ff8f66"
  },
  {
   "id": "gpt-6-luna",
   "display": "GPT-6 Luna",
   "effort": null,
   "vendor": "openai",
   "colour": "#5cc8ff"
  }
 ],
 "videos": [
  {
   "id": "yup",
   "title": "yup: 5 real bugs",
   "youtubeId": null,
   "publishedDate": "2026-11-11",
   "benchmarkId": "53bfd9666c98",
   "contenders": [
    "claude-haiku-5-5",
    "gpt-6-luna"
   ],
   "bonus": [],
   "pricingBasis": null,
   "totals": {
    "claude-haiku-5-5": {
     "fixed": 3,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 397.7,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.6,
       "low": 0.2667,
       "high": 0.9333
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.6
       },
       {
        "k": 2,
        "value": 0.7333
       },
       {
        "k": 3,
        "value": 0.8
       }
      ],
      "firstTryRate": 0.4,
      "feedbackGain": 0.2,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    },
    "gpt-6-luna": {
     "fixed": 2,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 348.6,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.4,
       "low": 0.0667,
       "high": 0.7333
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.4
       },
       {
        "k": 2,
        "value": 0.5333
       },
       {
        "k": 3,
        "value": 0.6
       }
      ],
      "firstTryRate": 0.2667,
      "feedbackGain": 0.1333,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    }
   }
  }
 ],
 "bugs": [
  {
   "scenarioId": "ac0e3c3a84a0",
   "videoIds": [
    "yup"
   ],
   "title": "feat: Implement standard schema interface (#2258)",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "871a74ff4739575d6a27ee21611e1a0f2c695c28",
   "fixCommit": "ced5f514a6033a96f5de3b4ae9c17fe0de86d68f",
   "benchmarkId": "53bfd9666c98",
   "links": {
    "dataset": {
     "task": "jquense__yup__ac0e3c3a84a0",
     "path": "harbor/jquense__yup__ac0e3c3a84a0"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/yup/claude-haiku-5-5/ac0e3c3a84a0.json",
      "replays/yup/claude-haiku-5-5/ac0e3c3a84a0.t2.json",
      "replays/yup/claude-haiku-5-5/ac0e3c3a84a0.t3.json"
     ],
     "gpt-6-luna": [
      "replays/yup/gpt-6-luna/ac0e3c3a84a0.json",
      "replays/yup/gpt-6-luna/ac0e3c3a84a0.t2.json",
      "replays/yup/gpt-6-luna/ac0e3c3a84a0.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "999d10d66163",
   "videoIds": [
    "yup"
   ],
   "title": "feat: resolve ref params if present when describing",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "d00abc331801388cb572a07094ba0cb803a50d5d",
   "fixCommit": "ef5303025c38e6e0dc0de53c990e7277cc74164e",
   "benchmarkId": "53bfd9666c98",
   "links": {
    "dataset": {
     "task": "jquense__yup__999d10d66163",
     "path": "harbor/jquense__yup__999d10d66163"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/yup/claude-haiku-5-5/999d10d66163.json",
      "replays/yup/claude-haiku-5-5/999d10d66163.t2.json",
      "replays/yup/claude-haiku-5-5/999d10d66163.t3.json"
     ],
     "gpt-6-luna": [
      "replays/yup/gpt-6-luna/999d10d66163.json",
      "replays/yup/gpt-6-luna/999d10d66163.t2.json",
      "replays/yup/gpt-6-luna/999d10d66163.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "f46c49cf80cc",
   "videoIds": [
    "yup"
   ],
   "title": "fix: lazy validation errors thrown in builders should resolve async like other validations",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "f27fa44a4509f84ee2274955de22581d6099ce0d",
   "fixCommit": "c7d7f977e02a7e578950dff192057e0b200999bd",
   "benchmarkId": "53bfd9666c98",
   "links": {
    "dataset": {
     "task": "jquense__yup__f46c49cf80cc",
     "path": "harbor/jquense__yup__f46c49cf80cc"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/yup/claude-haiku-5-5/f46c49cf80cc.json",
      "replays/yup/claude-haiku-5-5/f46c49cf80cc.t2.json",
      "replays/yup/claude-haiku-5-5/f46c49cf80cc.t3.json"
     ],
     "gpt-6-luna": [
      "replays/yup/gpt-6-luna/f46c49cf80cc.json",
      "replays/yup/gpt-6-luna/f46c49cf80cc.t2.json",
      "replays/yup/gpt-6-luna/f46c49cf80cc.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "c376f3ad043e",
   "videoIds": [
    "yup"
   ],
   "title": "feat(string): Create .datetime() (#2087)",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "ddea4e919cdf64cfa42c6891acd08c94c6972ae2",
   "fixCommit": "2a9e060594423018f517419ef5d2f10e417c9fbd",
   "benchmarkId": "53bfd9666c98",
   "links": {
    "dataset": {
     "task": "jquense__yup__c376f3ad043e",
     "path": "harbor/jquense__yup__c376f3ad043e"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/yup/claude-haiku-5-5/c376f3ad043e.json",
      "replays/yup/claude-haiku-5-5/c376f3ad043e.t2.json",
      "replays/yup/claude-haiku-5-5/c376f3ad043e.t3.json"
     ],
     "gpt-6-luna": [
      "replays/yup/gpt-6-luna/c376f3ad043e.json",
      "replays/yup/gpt-6-luna/c376f3ad043e.t2.json",
      "replays/yup/gpt-6-luna/c376f3ad043e.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "8fd9e38ae3b5",
   "videoIds": [
    "yup"
   ],
   "title": "Allow parent resolve before describe. (#2051)",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "6956ee788369dff00e5ecadb506726af3598a87e",
   "fixCommit": "020901fcf3cfb12a38afd90e4fe56e2e37c312a1",
   "benchmarkId": "53bfd9666c98",
   "links": {
    "dataset": {
     "task": "jquense__yup__8fd9e38ae3b5",
     "path": "harbor/jquense__yup__8fd9e38ae3b5"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/yup/claude-haiku-5-5/8fd9e38ae3b5.json",
      "replays/yup/claude-haiku-5-5/8fd9e38ae3b5.t2.json",
      "replays/yup/claude-haiku-5-5/8fd9e38ae3b5.t3.json"
     ],
     "gpt-6-luna": [
      "replays/yup/gpt-6-luna/8fd9e38ae3b5.json",
      "replays/yup/gpt-6-luna/8fd9e38ae3b5.t2.json",
      "replays/yup/gpt-6-luna/8fd9e38ae3b5.t3.json"
     ]
    }
   }
  }
 ],
 "runs": [
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "ac0e3c3a84a0",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 136.633,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/ac0e3c3a84a0.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "ad60537de292"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "999d10d66163",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 36.847,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/999d10d66163.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "ad60537de292"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "f46c49cf80cc",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 74.724,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/f46c49cf80cc.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "ad60537de292"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "c376f3ad043e",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 102.558,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/c376f3ad043e.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "ad60537de292"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "8fd9e38ae3b5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 30.019,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/8fd9e38ae3b5.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "ad60537de292"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "ac0e3c3a84a0",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 99.255,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/ac0e3c3a84a0.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2037a57cdd31"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "999d10d66163",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 54.869,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/999d10d66163.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2037a57cdd31"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "f46c49cf80cc",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 60.988,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/f46c49cf80cc.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2037a57cdd31"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "c376f3ad043e",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 111.199,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/c376f3ad043e.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2037a57cdd31"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "8fd9e38ae3b5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 49.017,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/8fd9e38ae3b5.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "2037a57cdd31"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "ac0e3c3a84a0",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 95.241,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/ac0e3c3a84a0.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "d8086bb4cd98"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "999d10d66163",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 48.969,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/999d10d66163.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "d8086bb4cd98"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "f46c49cf80cc",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 64.444,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/f46c49cf80cc.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "d8086bb4cd98"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "c376f3ad043e",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 160.339,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/c376f3ad043e.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "d8086bb4cd98"
  },
  {
   "videoId": "yup",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "8fd9e38ae3b5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 67.849,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/claude-haiku-5-5/8fd9e38ae3b5.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "ca2f73bca22a",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "d8086bb4cd98"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "ac0e3c3a84a0",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 121.282,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/ac0e3c3a84a0.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "dc9baf37fb0a"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "999d10d66163",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 66.54,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/999d10d66163.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "dc9baf37fb0a"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "f46c49cf80cc",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 72.921,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/f46c49cf80cc.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "dc9baf37fb0a"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "c376f3ad043e",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 79.06,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/c376f3ad043e.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "dc9baf37fb0a"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "8fd9e38ae3b5",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 70.529,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/8fd9e38ae3b5.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "dc9baf37fb0a"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "ac0e3c3a84a0",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 84.262,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/ac0e3c3a84a0.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "84268ac609b0"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "999d10d66163",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 36.916,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/999d10d66163.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "84268ac609b0"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "f46c49cf80cc",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 72.072,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/f46c49cf80cc.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "84268ac609b0"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "c376f3ad043e",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 78.816,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/c376f3ad043e.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "84268ac609b0"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "8fd9e38ae3b5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 38.003,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/8fd9e38ae3b5.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "84268ac609b0"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "ac0e3c3a84a0",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 110.967,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/ac0e3c3a84a0.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "14697a87466e"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "999d10d66163",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 35.352,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/999d10d66163.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "14697a87466e"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "f46c49cf80cc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 56.23,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/f46c49cf80cc.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "14697a87466e"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "c376f3ad043e",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 85.485,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/c376f3ad043e.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "14697a87466e"
  },
  {
   "videoId": "yup",
   "modelId": "gpt-6-luna",
   "scenarioId": "8fd9e38ae3b5",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 37.478,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/yup/gpt-6-luna/8fd9e38ae3b5.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "e89cc59f5aeb",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "14697a87466e"
  }
 ],
 "leaderboard": {
  "basis": "5 bugs, every trial counts; fixed = bugs resolved in at least half their trials",
  "bugCount": 5,
  "maxPoints": null,
  "omittedBugCount": 0,
  "rows": [
   {
    "modelId": "claude-haiku-5-5",
    "videoId": "yup",
    "fixed": 3,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 397.7,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.6,
      "low": 0.2667,
      "high": 0.9333
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.6
      },
      {
       "k": 2,
       "value": 0.7333
      },
      {
       "k": 3,
       "value": 0.8
      }
     ],
     "firstTryRate": 0.4,
     "feedbackGain": 0.2,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "ca2f73bca22a"
   },
   {
    "modelId": "gpt-6-luna",
    "videoId": "yup",
    "fixed": 2,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 348.6,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.4,
      "low": 0.0667,
      "high": 0.7333
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.4
      },
      {
       "k": 2,
       "value": 0.5333
      },
      {
       "k": 3,
       "value": 0.6
      }
     ],
     "firstTryRate": 0.2667,
     "feedbackGain": 0.1333,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "e89cc59f5aeb"
   }
  ]
 }
}