{
  "post": {
    "title": "AI agent reliability benchmarks, tracked",
    "url": "https://bardak.ai/posts/ai-agent-reliability-benchmarks",
    "updated": "2026-10-29T18:56:18.335Z"
  },
  "citation": "AI agent reliability benchmarks, tracked, Ivan Bardak, bardak.ai, https://bardak.ai/posts/ai-agent-reliability-benchmarks",
  "datasets": [
    {
      "key": "tau3-banking-share-of-tasks-solved-on-every-one-of-k-tries",
      "title": "τ³-Banking: share of tasks solved on every one of k tries",
      "type": "chart",
      "source": "τ-bench leaderboard (Sierra), τ³-Banking, standard setup, as of 27 October 2026",
      "caption": "Banking customer service with a 700-document knowledge base. Read along a line: Claude Opus 5 solves 48.7% of tasks on one try, but only 32.0% on four tries in a row. The four best models, all released in July 2026, under the same test conditions run by Sierra.",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/tau3-banking-share-of-tasks-solved-on-every-one-of-k-tries.csv",
      "header": [
        "k",
        "Qwen 3.8 Max",
        "Claude Opus 5",
        "Grok 4.5",
        "GPT-5.6-sol"
      ],
      "rows": [
        [
          "1",
          "55.2",
          "48.7",
          "47.9",
          "46.9"
        ],
        [
          "2",
          "45.2",
          "39.2",
          "39.5",
          "37.3"
        ],
        [
          "3",
          "39.7",
          "34.5",
          "35.3",
          "32.0"
        ],
        [
          "4",
          "35.1",
          "32.0",
          "32.0",
          "27.8"
        ]
      ]
    },
    {
      "key": "tau2-bench-online-shop-tasks-solved-on-one-try-and-on-four-in-a-row",
      "title": "τ²-bench online shop: tasks solved on one try and on four in a row",
      "type": "benchmarkTable",
      "source": "τ-bench leaderboard (Sierra), τ²-bench v1.0.1, standard setup, runs from February to May 2026",
      "caption": "",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/tau2-bench-online-shop-tasks-solved-on-one-try-and-on-four-in-a-row.csv",
      "header": [
        "Model",
        "1 try",
        "4 in a row"
      ],
      "rows": [
        [
          "Qwen3.5-397B",
          "84.4%",
          "59.7%"
        ],
        [
          "GPT-5.2",
          "81.6%",
          "51.8%"
        ],
        [
          "Claude Opus 4.5",
          "79.6%",
          "51.8%"
        ]
      ]
    },
    {
      "key": "tau2-bench-airline-bookings-tasks-solved-on-one-try-and-on-four-in-a-row",
      "title": "τ²-bench airline bookings: tasks solved on one try and on four in a row",
      "type": "benchmarkTable",
      "source": "τ-bench leaderboard (Sierra), τ²-bench v1.0.1, standard setup, runs from February to May 2026",
      "caption": "These domains have results only for models released up to February 2026. Two and three tries in a row are on the leaderboard. Compare models within a domain, not across.",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/tau2-bench-airline-bookings-tasks-solved-on-one-try-and-on-four-in-a-row.csv",
      "header": [
        "Model",
        "1 try",
        "4 in a row"
      ],
      "rows": [
        [
          "GPT-5.2",
          "83.0%",
          "72.0%"
        ],
        [
          "Claude Opus 4.5",
          "84.0%",
          "70.0%"
        ],
        [
          "Qwen3.5-397B",
          "81.5%",
          "68.0%"
        ]
      ]
    },
    {
      "key": "task-length-agents-complete-half-the-time-and-four-times-out-of-five",
      "title": "Task length agents complete half the time and four times out of five",
      "type": "chart",
      "source": "METR, Time horizons of frontier AI models (Time Horizon 1.1), last updated 8 September 2026",
      "caption": "Read a pair of bars: Claude Opus 4.6 succeeds half the time on tasks that take a person about 12 hours, but four times out of five only on tasks of about 70 minutes. Each estimate has a wide margin of error, and METR calls results above 16 hours unreliable, which covers the 50% figure for Claude Mythos Preview (about 17 hours).",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/task-length-agents-complete-half-the-time-and-four-times-out-of-five.csv",
      "header": [
        "model",
        "50% success",
        "80% success"
      ],
      "rows": [
        [
          "Claude Mythos Preview (early)",
          "1045",
          "186"
        ],
        [
          "Claude Opus 4.6",
          "719",
          "70"
        ],
        [
          "Gemini 3.1 Pro",
          "384",
          "90"
        ],
        [
          "GPT-5.4",
          "342",
          "54"
        ],
        [
          "Claude Opus 4.5",
          "293",
          "49"
        ],
        [
          "o3",
          "120",
          "30"
        ],
        [
          "Claude 3.7 Sonnet",
          "60",
          "12"
        ]
      ]
    },
    {
      "key": "theagentcompany-share-of-tasks-fully-completed",
      "title": "TheAgentCompany: share of tasks fully completed",
      "type": "chart",
      "source": "TheAgentCompany leaderboard (Carnegie Mellon University)",
      "caption": "Two entries with higher scores (43% and 41%) used a different simulated environment and are not comparable. The leaderboard has had no new checked entries since November 2025, so no 2026 model appears here.",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/theagentcompany-share-of-tasks-fully-completed.csv",
      "header": [
        "agent",
        "Tasks fully completed"
      ],
      "rows": [
        [
          "OpenHands-Versa + Claude Sonnet 4",
          "33.1"
        ],
        [
          "OpenHands-Versa + Claude 3.7 Sonnet",
          "30.9"
        ],
        [
          "OpenHands + Gemini 2.5 Pro",
          "30.3"
        ],
        [
          "OpenHands + Claude 3.7 Sonnet",
          "26.3"
        ],
        [
          "OpenHands + Claude 3.5 Sonnet",
          "24.0"
        ],
        [
          "OpenHands + GPT-4o",
          "8.6"
        ]
      ]
    },
    {
      "key": "automationbench-objectives-completed-and-the-score-once-rule-breaks-count",
      "title": "AutomationBench: objectives completed, and the score once rule breaks count",
      "type": "chart",
      "source": "Artificial Analysis, AutomationBench-AA, accessed 27 October 2026",
      "caption": "Read a pair of bars: Claude Opus 5.5 completes 90.4% of the objectives, but scores 69.5% once tasks where it broke a rule count as zero. Broken rules per task range from 0.31 (Gemini 4 Argon) to 0.71 (Claude Fable 5.1); 0.42 for Opus 5.5 and 0.44 for GPT-6 Astra. Models released September 2026, at the highest reasoning setting Artificial Analysis lists (high for Gemini), one run per task.",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/automationbench-objectives-completed-and-the-score-once-rule-breaks-count.csv",
      "header": [
        "model",
        "Objectives completed",
        "Score when a broken rule zeroes the task"
      ],
      "rows": [
        [
          "Gemini 4 Argon",
          "93.5",
          "77.5"
        ],
        [
          "Claude Sonnet 5.5",
          "91.0",
          "71.8"
        ],
        [
          "Claude Opus 5.5",
          "90.4",
          "69.5"
        ],
        [
          "GPT-6 Astra",
          "88.8",
          "68.5"
        ],
        [
          "GPT-6.1 Sol",
          "88.4",
          "64.9"
        ],
        [
          "Claude Fable 5.1",
          "88.1",
          "59.4"
        ]
      ]
    },
    {
      "key": "benchmark-terms-in-plain-language",
      "title": "Benchmark terms in plain language",
      "type": "benchmarkTable",
      "source": "",
      "caption": "",
      "csv": "https://bardak.ai/data/ai-agent-reliability-benchmarks/benchmark-terms-in-plain-language.csv",
      "header": [
        "Term",
        "Meaning"
      ],
      "rows": [
        [
          "Benchmark",
          "A standard test that every model takes under the same conditions"
        ],
        [
          "Agent",
          "An AI model that takes actions with tools, not only answers questions"
        ],
        [
          "pass^k",
          "The share of tasks an agent solves on every one of k tries"
        ],
        [
          "pass@k",
          "The share of tasks it solves on at least one of k tries; flattering, and not what production needs"
        ],
        [
          "Simulated user",
          "Another AI model playing the customer, so every agent faces the same conversation"
        ],
        [
          "Time horizon",
          "The length of task, in a skilled person's working time, that a model completes with a given success rate"
        ],
        [
          "Leaderboard",
          "The benchmark owner's official table of results"
        ],
        [
          "Guardrail",
          "A condition that holds at the start of a task and must still hold at the end, such as other customers' records left unchanged"
        ]
      ]
    }
  ]
}