{ "post": { "title": "Agentic AI from demo to production: what the evidence says about the architecture", "url": "https://bardak.ai/posts/agentic-ai-from-demo-to-production", "updated": "2026-10-29T18:55:29.077Z" }, "citation": "Agentic AI from demo to production: what the evidence says about the architecture, Ivan Bardak, bardak.ai, https://bardak.ai/posts/agentic-ai-from-demo-to-production", "datasets": [ { "key": "ai-agent-architecture-the-parts-and-what-breaks-in-production", "title": "AI agent architecture: the parts and what breaks in production", "type": "benchmarkTable", "source": "", "caption": "", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/ai-agent-architecture-the-parts-and-what-breaks-in-production.csv", "header": [ "Part", "Its job", "What breaks in production" ], "rows": [ [ "Model", "Plans the steps, chooses tools and writes the output", "Rarely what holds a pilot back; output quality, security and cost do" ], [ "Tools", "The actions the agent can take, through APIs or MCP servers", "Existing endpoints exposed wholesale, with no notion of intent" ], [ "Identity and access", "Whose rights the agent acts with, and which ones", "Broad permissions meeting untrusted content caused the worst published incidents" ], [ "Retrieval and memory", "What the agent can find in your data", "Vector search alone misses identifiers; hybrid search with reranking does best" ], [ "Reversibility", "Which actions can be undone, and who approves the rest", "Prompt instructions did not stop destructive actions; environments and approvals did" ], [ "Measurement", "How you know it works, release after release", "Single-try success overstates reliability; track pass^k and cost per task" ] ] }, { "key": "what-the-surveys-say-and-their-caveats", "title": "What the surveys say, and their caveats", "type": "benchmarkTable", "source": "McKinsey 2026; Gartner, October 2026 and June 2025; MIT NANDA 2025; LangChain 2025", "caption": "Read each finding with its sample. None of these is a random sample of companies.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/what-the-surveys-say-and-their-caveats.csv", "header": [ "Source", "Who was asked", "Finding", "Caveat" ], "rows": [ [ "McKinsey, The state of AI in 2026", "1,719 respondents in 97 countries, May to June 2026", "40% of organisations with over $1B revenue say they are at least scaling agents, up from 27% a year earlier; 22% of smaller ones", "Self-reported; scaling can mean one business function" ], [ "Gartner CIO survey, October 2026", "2,002 CIOs, May to July 2026", "13% say their enterprise has realised significant or transformational value from AI; only 31% have guidelines for when a pilot should not move to production", "About AI in general, not only agents" ], [ "Gartner prediction, June 2025", "Analyst forecast", "Over 40% of agentic AI projects will be canceled by the end of 2027, because of cost, unclear business value or inadequate risk controls", "A prediction, not a measurement" ], [ "MIT NANDA, July 2025", "300+ public initiatives, 52 interviews, 153 survey responses", "5% of task-specific AI systems reached production with a sustained productivity or P&L impact", "Preliminary and directional, based on interviews" ], [ "LangChain, December 2025", "1,340 respondents, 63% from technology companies", "57% have agents in production; quality of outputs is the top blocker (33%), then latency (20%) and security (16%)", "The audience of an agent framework vendor" ] ] }, { "key": "mit-nanda-share-of-organisations-at-each-stage", "title": "MIT NANDA: share of organisations at each stage", "type": "chart", "source": "MIT NANDA, The GenAI Divide: State of AI in Business 2025 (preliminary findings, July 2025)", "caption": "Successfully implemented means users or executives described a marked and sustained productivity or P&L impact. The authors call the figures directional: they come from interviews, not company reporting.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/mit-nanda-share-of-organisations-at-each-stage.csv", "header": [ "stage", "General-purpose tools", "Task-specific systems" ], "rows": [ [ "Investigated", "80", "60" ], [ "Piloted", "50", "20" ], [ "Successfully implemented", "40", "5" ] ] }, { "key": "two-incidents-one-pattern-the-lethal-trifecta", "title": "Two incidents, one pattern: the lethal trifecta", "type": "knowledgeGraph", "source": "Invariant Labs, May 2025; General Analysis, July 2025; Simon Willison, June 2025", "caption": "Remove any one leg, by scope, by environment or by an approval step, and the attack no longer works.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/two-incidents-one-pattern-the-lethal-trifecta.csv", "header": [ "source", "target", "relation" ], "rows": [ [ "GitHub MCP incident", "Private data", "read private repositories" ], [ "GitHub MCP incident", "Untrusted content", "read a public issue" ], [ "GitHub MCP incident", "External communication", "opened a public pull request" ], [ "Supabase MCP incident", "Private data", "read a tokens table as service_role" ], [ "Supabase MCP incident", "Untrusted content", "read a support ticket" ], [ "Supabase MCP incident", "External communication", "wrote into the ticket thread" ] ] }, { "key": "retrieval-quality-by-method-and-query-type", "title": "Retrieval quality by method and query type", "type": "chart", "source": "Microsoft, Azure AI Search: Outperforming vector search with hybrid retrieval and ranking capabilities (2023); four customer datasets", "caption": "Keyword queries here are short queries made only of identifier words. On them, vector search alone scored 11.7 against 79.2 for keyword search. Hybrid retrieval with reranking was the best or close to it on every query type.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/retrieval-quality-by-method-and-query-type.csv", "header": [ "method", "All queries", "Keyword queries", "Concept-seeking queries" ], "rows": [ [ "Keyword (BM25)", "40.6", "79.2", "39.0" ], [ "Vector", "43.8", "11.7", "45.8" ], [ "Hybrid", "48.4", "61.0", "46.3" ], [ "Hybrid + semantic ranker", "60.1", "66.9", "59.6" ] ] }, { "key": "tau3-banking-tasks-solved-on-every-one-of-k-tries", "title": "τ³-Banking: tasks solved on every one of k tries", "type": "chart", "source": "τ-bench leaderboard (τ³-Banking, standard setup, four trials per task), as of October 2026", "caption": "Banking customer service with a 700-document knowledge base to search. The four best models, all released in July 2026, solve 47 to 55% of tasks on one try, but only 28 to 35% on all four. Same test conditions for all four, run by Sierra.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/tau3-banking-tasks-solved-on-every-one-of-k-tries.csv", "header": [ "k", "Qwen 3.8 Max", "Claude Opus 5", "Grok 4.5", "GPT-5.6-sol" ], "rows": [ [ "1", "55.2", "48.7", "47.9", "46.9" ], [ "2", "45.2", "39.2", "39.5", "37.3" ], [ "3", "39.7", "34.5", "35.3", "32.0" ], [ "4", "35.1", "32.0", "32.0", "27.8" ] ] }, { "key": "six-questions-before-an-agent-pilot", "title": "Six questions before an agent pilot", "type": "benchmarkTable", "source": "", "caption": "The sixth question is the one most teams skip, and the one that saves the most money.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/six-questions-before-an-agent-pilot.csv", "header": [ "Question", "Why it matters", "A good answer" ], "rows": [ [ "Which tools will the agent call, and who owns their contracts?", "Tool design changes agent accuracy more than prompt wording", "A short list of task-level tools with typed inputs and named owners" ], [ "How does the user delegate access, and which scopes are the minimum?", "The published leaks came from broad permissions plus untrusted content", "OAuth with per-capability scopes, tokens bound to one server, no shared admin keys" ], [ "What does retrieval look like today, and how is relevance measured?", "Vector search alone fails on identifiers; hybrid with reranking does best", "A labelled query set and a relevance metric tracked per release" ], [ "Which actions are irreversible, and where does a human approve them?", "Prompt instructions did not stop the Replit deletion; environments did", "Drafts by default, gated production, tested restore" ], [ "What are the success metrics, and who reviews them weekly?", "Single-try success overstates reliability by 25 to 30 points", "Task success and pass^k on a fixed set, cost per task, undo rate" ], [ "What result makes us stop?", "Only 31% of CIOs have rules for when a pilot should not move to production", "A go or no-go threshold agreed before the pilot starts" ] ] }, { "key": "appraising-the-evidence", "title": "Appraising the evidence", "type": "benchmarkTable", "source": "", "caption": "No single source here settles the question. The five areas hold because different kinds of evidence point the same way.", "csv": "https://bardak.ai/data/agentic-ai-from-demo-to-production/appraising-the-evidence.csv", "header": [ "Evidence", "Strength", "Main limitation" ], "rows": [ [ "Surveys (McKinsey, Gartner, MIT NANDA, LangChain)", "Moderate for direction, weak for exact numbers", "Self-reported, non-random samples, different definitions of production and success" ], [ "Published incidents (GitHub MCP, Supabase MCP, Replit)", "Strong as proof that a failure mode exists", "Demonstrations and single cases; they say nothing about how often it happens" ], [ "Engineering guidance (Anthropic, Microsoft)", "Useful practice from teams with real systems", "Vendors writing about their own products and benchmarks" ], [ "Benchmarks (τ-bench, METR, TheAgentCompany)", "Strong, repeatable measurement", "Simulated tasks and users; your workload will differ" ], [ "My own projects", "Direct experience", "One practitioner, measured by the same team that built it" ] ] } ] }