{
  "post": {
    "title": "Does AI make developers faster? What the controlled studies say",
    "url": "https://bardak.ai/posts/does-ai-make-developers-faster",
    "updated": "2026-10-29T18:56:12.858Z"
  },
  "citation": "Does AI make developers faster? What the controlled studies say, Ivan Bardak, bardak.ai, https://bardak.ai/posts/does-ai-make-developers-faster",
  "datasets": [
    {
      "key": "controlled-studies-of-ai-and-developer-speed",
      "title": "Controlled studies of AI and developer speed",
      "type": "benchmarkTable",
      "source": "Peng et al. 2023; Paradis et al. 2024; Cui et al. 2025; METR 2025 and 2026",
      "caption": "Different studies measure different things: time on a task, or completed tasks over months. Read each number with its setting.",
      "csv": "https://bardak.ai/data/does-ai-make-developers-faster/controlled-studies-of-ai-and-developer-speed.csv",
      "header": [
        "Study",
        "Who and what",
        "Effect",
        "95% interval",
        "Setting"
      ],
      "rows": [
        [
          "Peng et al. 2023",
          "95 freelance developers; write an HTTP server in JavaScript",
          "55.8% less time",
          "21% to 89%",
          "One small task from scratch, GitHub Copilot"
        ],
        [
          "Paradis et al. 2024",
          "96 Google engineers; one complex enterprise task",
          "About 21% less time",
          "Wide (the authors' word)",
          "Lab task on Google's internal AI features, summer 2024"
        ],
        [
          "Cui et al. 2025",
          "4,867 developers at Microsoft, Accenture and a Fortune 100 company",
          "26% more completed tasks",
          "Standard error 10.3 points",
          "Field experiments over months of normal work, GitHub Copilot"
        ],
        [
          "METR 2025",
          "16 experienced open-source maintainers; 246 issues in their own repositories",
          "19% more time",
          "+2% to +39%",
          "Real issues, early 2025, mostly Cursor with Claude 3.5 and 3.7 Sonnet"
        ],
        [
          "METR 2026",
          "57 developers, 800+ tasks, 143 repositories",
          "18% less time (returning); 4% less (new)",
          "-38% to +9%; -15% to +9%",
          "Same design a year later; METR calls it weak evidence"
        ]
      ]
    },
    {
      "key": "metr-2025-what-developers-expected-felt-and-measured",
      "title": "METR 2025: what developers expected, felt and measured",
      "type": "chart",
      "source": "METR, Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity (July 2025)",
      "caption": "The developers expected AI to make them 24% faster and still believed it had made them 20% faster after finishing. The measured effect went the other way.",
      "csv": "https://bardak.ai/data/does-ai-make-developers-faster/metr-2025-what-developers-expected-felt-and-measured.csv",
      "header": [
        "measure",
        "Change in task time"
      ],
      "rows": [
        [
          "Expected before the study",
          "-24"
        ],
        [
          "Believed after the study",
          "-20"
        ],
        [
          "Measured",
          "19"
        ]
      ]
    },
    {
      "key": "change-in-task-time-with-ai-by-study-negative-means-faster",
      "title": "Change in task time with AI, by study (negative means faster)",
      "type": "chart",
      "source": "Peng et al. 2023; Paradis et al. 2024; METR 2025 and 2026",
      "caption": "Only the studies that measured time on task. Cui et al. measured completed tasks over months, a different quantity, so it is not on this chart. Intervals are in the table above; three of these five are wide or include zero.",
      "csv": "https://bardak.ai/data/does-ai-make-developers-faster/change-in-task-time-with-ai-by-study-negative-means-faster.csv",
      "header": [
        "study",
        "Change in task time"
      ],
      "rows": [
        [
          "Peng et al. 2023: small task from scratch",
          "-55.8"
        ],
        [
          "Paradis et al. 2024: Google enterprise task",
          "-21"
        ],
        [
          "METR 2025: experienced maintainers",
          "19"
        ],
        [
          "METR 2026: returning developers",
          "-18"
        ],
        [
          "METR 2026: new developers",
          "-4"
        ]
      ]
    },
    {
      "key": "appraising-the-evidence",
      "title": "Appraising the evidence",
      "type": "benchmarkTable",
      "source": "",
      "caption": "My appraisal. Note that three of the five studies have authors from companies that build or sell AI coding tools; METR is an independent non-profit.",
      "csv": "https://bardak.ai/data/does-ai-make-developers-faster/appraising-the-evidence.csv",
      "header": [
        "Study",
        "Strength",
        "Main limitation"
      ],
      "rows": [
        [
          "Peng et al. 2023",
          "Strong design, narrow scope",
          "One small task from scratch; freelancers; no quality measure; authors from GitHub and Microsoft"
        ],
        [
          "Paradis et al. 2024",
          "Strong design, one company",
          "One lab task on internal tooling; wide interval; authors from Google"
        ],
        [
          "Cui et al. 2025",
          "Strongest for organisations",
          "Pooled estimate is clear but each experiment is noisy; completed tasks is a proxy for value"
        ],
        [
          "METR 2025",
          "Strong design, small sample",
          "16 developers, all highly experienced in their own repositories; early-2025 tools"
        ],
        [
          "METR 2026",
          "Weak, by the authors' own account",
          "Selection effects: developers refused or held back tasks; time tracking unreliable with agents"
        ]
      ]
    }
  ]
}