{
 "schemaVersion": 1,
 "generatedAt": "2026-11-11T21:01:58Z",
 "repo": {
  "display": "cli",
  "site": "https://github.com/urfave/cli",
  "language": "go",
  "url": "https://github.com/urfave/cli",
  "visibility": "public"
 },
 "method": {
  "oneTry": false,
  "reproduceFirst": false,
  "hiddenTests": true,
  "weightRule": null,
  "totalPoints": null,
  "trialsPerBug": 3
 },
 "models": [
  {
   "id": "claude-haiku-5-5",
   "display": "Claude Haiku 5.5",
   "effort": null,
   "vendor": "anthropic",
   "colour": "#ff8f66"
  },
  {
   "id": "gpt-6-luna",
   "display": "GPT-6 Luna",
   "effort": null,
   "vendor": "openai",
   "colour": "#5cc8ff"
  }
 ],
 "videos": [
  {
   "id": "urfave-cli",
   "title": "urfave/cli: 5 real bugs",
   "youtubeId": null,
   "publishedDate": "2026-11-11",
   "benchmarkId": "1230d2585451",
   "contenders": [
    "claude-haiku-5-5",
    "gpt-6-luna"
   ],
   "bonus": [],
   "pricingBasis": null,
   "totals": {
    "claude-haiku-5-5": {
     "fixed": 5,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 338.1,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 1,
       "low": 1,
       "high": 1
      },
      "passAtK": [
       {
        "k": 1,
        "value": 1
       },
       {
        "k": 2,
        "value": 1
       },
       {
        "k": 3,
        "value": 1
       }
      ],
      "firstTryRate": 0.9333,
      "feedbackGain": 0.0667,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    },
    "gpt-6-luna": {
     "fixed": 4,
     "attempted": 5,
     "points": null,
     "maxPoints": null,
     "costUSD": null,
     "timeSeconds": 308.9,
     "tokens": null,
     "statistics": {
      "bugCount": 5,
      "trialsPerBug": 3,
      "trialCount": 15,
      "resolveRate": {
       "mean": 0.8667,
       "low": 0.6,
       "high": 1
      },
      "passAtK": [
       {
        "k": 1,
        "value": 0.8667
       },
       {
        "k": 2,
        "value": 0.9333
       },
       {
        "k": 3,
        "value": 1
       }
      ],
      "firstTryRate": 0.8,
      "feedbackGain": 0.0667,
      "confidence": 0.95,
      "bootstrapSamples": 2000,
      "seed": 166
     }
    }
   }
  }
 ],
 "bugs": [
  {
   "scenarioId": "6308eb3cb2be",
   "videoIds": [
    "urfave-cli"
   ],
   "title": "fix: count runes, not bytes, when wrapping help text",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "d854bb40d9441bd4bfba5eee6e33034d9e4f47e4",
   "fixCommit": "ca8f45a916f9fdfa14b21238d43ebc972fbfcd26",
   "benchmarkId": "1230d2585451",
   "links": {
    "dataset": {
     "task": "urfave__cli__6308eb3cb2be",
     "path": "harbor/urfave__cli__6308eb3cb2be"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/urfave-cli/claude-haiku-5-5/6308eb3cb2be.json",
      "replays/urfave-cli/claude-haiku-5-5/6308eb3cb2be.t2.json",
      "replays/urfave-cli/claude-haiku-5-5/6308eb3cb2be.t3.json"
     ],
     "gpt-6-luna": [
      "replays/urfave-cli/gpt-6-luna/6308eb3cb2be.json",
      "replays/urfave-cli/gpt-6-luna/6308eb3cb2be.t2.json",
      "replays/urfave-cli/gpt-6-luna/6308eb3cb2be.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "744aecdc3087",
   "videoIds": [
    "urfave-cli"
   ],
   "title": "fix: omit ancestor flags when a fish command skips flag parsing",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "7b834860b8afb13bc311cb1ca420e488a9facfc2",
   "fixCommit": "ce39339c4736717d043c160a8368a417b965130b",
   "benchmarkId": "1230d2585451",
   "links": {
    "dataset": {
     "task": "urfave__cli__744aecdc3087",
     "path": "harbor/urfave__cli__744aecdc3087"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/urfave-cli/claude-haiku-5-5/744aecdc3087.json",
      "replays/urfave-cli/claude-haiku-5-5/744aecdc3087.t2.json",
      "replays/urfave-cli/claude-haiku-5-5/744aecdc3087.t3.json"
     ],
     "gpt-6-luna": [
      "replays/urfave-cli/gpt-6-luna/744aecdc3087.json",
      "replays/urfave-cli/gpt-6-luna/744aecdc3087.t2.json",
      "replays/urfave-cli/gpt-6-luna/744aecdc3087.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "16a42a1f67ca",
   "videoIds": [
    "urfave-cli"
   ],
   "title": "fix: show help for the whole subcommand path given to help",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "293b0229b99c460fa66283418569773681df010d",
   "fixCommit": "1d8baf6f262296e150c1651bf86528aebf53dc2b",
   "benchmarkId": "1230d2585451",
   "links": {
    "dataset": {
     "task": "urfave__cli__16a42a1f67ca",
     "path": "harbor/urfave__cli__16a42a1f67ca"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/urfave-cli/claude-haiku-5-5/16a42a1f67ca.json",
      "replays/urfave-cli/claude-haiku-5-5/16a42a1f67ca.t2.json",
      "replays/urfave-cli/claude-haiku-5-5/16a42a1f67ca.t3.json"
     ],
     "gpt-6-luna": [
      "replays/urfave-cli/gpt-6-luna/16a42a1f67ca.json",
      "replays/urfave-cli/gpt-6-luna/16a42a1f67ca.t2.json",
      "replays/urfave-cli/gpt-6-luna/16a42a1f67ca.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "8e2ad1d071fc",
   "videoIds": [
    "urfave-cli"
   ],
   "title": "fix: use the configured completion command name in its help",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "6898360ab3624054a58cd2cc2e90d1724f8eed1e",
   "fixCommit": "c212c5ca4b62a9330a3b83bea35c233e7d2d4d57",
   "benchmarkId": "1230d2585451",
   "links": {
    "dataset": {
     "task": "urfave__cli__8e2ad1d071fc",
     "path": "harbor/urfave__cli__8e2ad1d071fc"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/urfave-cli/claude-haiku-5-5/8e2ad1d071fc.json",
      "replays/urfave-cli/claude-haiku-5-5/8e2ad1d071fc.t2.json",
      "replays/urfave-cli/claude-haiku-5-5/8e2ad1d071fc.t3.json"
     ],
     "gpt-6-luna": [
      "replays/urfave-cli/gpt-6-luna/8e2ad1d071fc.json",
      "replays/urfave-cli/gpt-6-luna/8e2ad1d071fc.t2.json",
      "replays/urfave-cli/gpt-6-luna/8e2ad1d071fc.t3.json"
     ]
    }
   }
  },
  {
   "scenarioId": "57e54d950757",
   "videoIds": [
    "urfave-cli"
   ],
   "title": "fix: offer the persistent flags of ancestors in the fish completion script",
   "plainTitle": null,
   "explanation": null,
   "points": null,
   "linesChanged": null,
   "inCommonBoard": true,
   "buggyCommit": "6898360ab3624054a58cd2cc2e90d1724f8eed1e",
   "fixCommit": "7b834860b8afb13bc311cb1ca420e488a9facfc2",
   "benchmarkId": "1230d2585451",
   "links": {
    "dataset": {
     "task": "urfave__cli__57e54d950757",
     "path": "harbor/urfave__cli__57e54d950757"
    },
    "video": null,
    "replays": {
     "claude-haiku-5-5": [
      "replays/urfave-cli/claude-haiku-5-5/57e54d950757.json",
      "replays/urfave-cli/claude-haiku-5-5/57e54d950757.t2.json",
      "replays/urfave-cli/claude-haiku-5-5/57e54d950757.t3.json"
     ],
     "gpt-6-luna": [
      "replays/urfave-cli/gpt-6-luna/57e54d950757.json",
      "replays/urfave-cli/gpt-6-luna/57e54d950757.t2.json",
      "replays/urfave-cli/gpt-6-luna/57e54d950757.t3.json"
     ]
    }
   }
  }
 ],
 "runs": [
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "6308eb3cb2be",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 57.021,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/6308eb3cb2be.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "d681ddf35a4f"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "744aecdc3087",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 35.439,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/744aecdc3087.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "d681ddf35a4f"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "16a42a1f67ca",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 78.626,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/16a42a1f67ca.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "d681ddf35a4f"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "8e2ad1d071fc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 41.293,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/8e2ad1d071fc.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "d681ddf35a4f"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "57e54d950757",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 104.366,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/57e54d950757.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "d681ddf35a4f"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "6308eb3cb2be",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 43.225,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/6308eb3cb2be.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "a46935173563"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "744aecdc3087",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 42.089,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/744aecdc3087.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "a46935173563"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "16a42a1f67ca",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 154.002,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/16a42a1f67ca.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "a46935173563"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "8e2ad1d071fc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 68.226,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/8e2ad1d071fc.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "a46935173563"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "57e54d950757",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 115.674,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/57e54d950757.t2.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "a46935173563"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "6308eb3cb2be",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 40.088,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/6308eb3cb2be.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "dba40c218afc"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "744aecdc3087",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 41.905,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/744aecdc3087.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "dba40c218afc"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "16a42a1f67ca",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 70.735,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/16a42a1f67ca.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "dba40c218afc"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "8e2ad1d071fc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 39.485,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/8e2ad1d071fc.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "dba40c218afc"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "claude-haiku-5-5",
   "scenarioId": "57e54d950757",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 82.143,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/claude-haiku-5-5/57e54d950757.t3.json",
   "agentCli": "claude-code",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "7518a4779300",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "dba40c218afc"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "6308eb3cb2be",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 48.823,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/6308eb3cb2be.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "9c1a86c66868"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "744aecdc3087",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 55.932,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/744aecdc3087.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "9c1a86c66868"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "16a42a1f67ca",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 61.737,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/16a42a1f67ca.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "9c1a86c66868"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "8e2ad1d071fc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 52.626,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/8e2ad1d071fc.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "9c1a86c66868"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "57e54d950757",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 95.906,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/57e54d950757.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 1,
   "trialCount": 3,
   "runId": "9c1a86c66868"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "6308eb3cb2be",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 41.834,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/6308eb3cb2be.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "42a398c302e9"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "744aecdc3087",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 41.613,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/744aecdc3087.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "42a398c302e9"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "16a42a1f67ca",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 53.635,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/16a42a1f67ca.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "42a398c302e9"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "8e2ad1d071fc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 54.606,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/8e2ad1d071fc.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "42a398c302e9"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "57e54d950757",
   "passed": false,
   "reproducedFirst": null,
   "durationSeconds": 111.95,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/57e54d950757.t2.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 2,
   "trialCount": 3,
   "runId": "42a398c302e9"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "6308eb3cb2be",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 42.023,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/6308eb3cb2be.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "491f63d3cf6b"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "744aecdc3087",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 42.027,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/744aecdc3087.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "491f63d3cf6b"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "16a42a1f67ca",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 60.378,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/16a42a1f67ca.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "491f63d3cf6b"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "8e2ad1d071fc",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 51.024,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/8e2ad1d071fc.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "491f63d3cf6b"
  },
  {
   "videoId": "urfave-cli",
   "modelId": "gpt-6-luna",
   "scenarioId": "57e54d950757",
   "passed": true,
   "reproducedFirst": null,
   "durationSeconds": 112.566,
   "tokens": null,
   "costUSD": null,
   "timedOut": false,
   "blindOverall": null,
   "replay": "replays/urfave-cli/gpt-6-luna/57e54d950757.t3.json",
   "agentCli": "codex",
   "agentCliSource": "RunDocument.agent",
   "runSetId": "b4190fa7f2a1",
   "trialIndex": 3,
   "trialCount": 3,
   "runId": "491f63d3cf6b"
  }
 ],
 "leaderboard": {
  "basis": "5 bugs, every trial counts; fixed = bugs resolved in at least half their trials",
  "bugCount": 5,
  "maxPoints": null,
  "omittedBugCount": 0,
  "rows": [
   {
    "modelId": "claude-haiku-5-5",
    "videoId": "urfave-cli",
    "fixed": 5,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 338.1,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 1,
      "low": 1,
      "high": 1
     },
     "passAtK": [
      {
       "k": 1,
       "value": 1
      },
      {
       "k": 2,
       "value": 1
      },
      {
       "k": 3,
       "value": 1
      }
     ],
     "firstTryRate": 0.9333,
     "feedbackGain": 0.0667,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "7518a4779300"
   },
   {
    "modelId": "gpt-6-luna",
    "videoId": "urfave-cli",
    "fixed": 4,
    "of": 5,
    "points": null,
    "costUSD": null,
    "timeSeconds": 308.9,
    "tokens": null,
    "runCount": 5,
    "statistics": {
     "bugCount": 5,
     "trialsPerBug": 3,
     "trialCount": 15,
     "resolveRate": {
      "mean": 0.8667,
      "low": 0.6,
      "high": 1
     },
     "passAtK": [
      {
       "k": 1,
       "value": 0.8667
      },
      {
       "k": 2,
       "value": 0.9333
      },
      {
       "k": 3,
       "value": 1
      }
     ],
     "firstTryRate": 0.8,
     "feedbackGain": 0.0667,
     "confidence": 0.95,
     "bootstrapSamples": 2000,
     "seed": 166
    },
    "trials": 3,
    "runSetId": "b4190fa7f2a1"
   }
  ]
 }
}