{
  "post": {
    "title": "NVIDIA Auto Ontology vs semantic layers: will data agents get the right number?",
    "url": "https://bardak.ai/posts/nvidia-auto-ontology",
    "updated": "2026-11-08T08:05:05.192Z"
  },
  "citation": "NVIDIA Auto Ontology vs semantic layers: will data agents get the right number?, Ivan Bardak, bardak.ai, https://bardak.ai/posts/nvidia-auto-ontology",
  "datasets": [
    {
      "key": "how-auto-ontology-sits-between-data-and-agents",
      "title": "How Auto Ontology sits between data and agents",
      "type": "knowledgeGraph",
      "source": "NVIDIA/auto-ontology README and docs/mcp.md, read 8 November 2026",
      "caption": "The language model drafts the catalog and also writes the SQL; nothing in the query path is a deterministic compiler.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/how-auto-ontology-sits-between-data-and-agents.csv",
      "header": [
        "source",
        "target",
        "relation"
      ],
      "rows": [
        [
          "Databases",
          "Profiler",
          "metadata and sample values"
        ],
        [
          "Profiler",
          "LLM semantic compile",
          "profiles"
        ],
        [
          "LLM semantic compile",
          "Catalog",
          "terms, SQL snippets, inferred joins"
        ],
        [
          "Data experts",
          "Catalog",
          "review and certify"
        ],
        [
          "Catalog",
          "Apache Ossie YAML",
          "export and import"
        ],
        [
          "Catalog",
          "Text-to-SQL agent",
          "business context"
        ],
        [
          "Text-to-SQL agent",
          "Databases",
          "generated SQL"
        ],
        [
          "Text-to-SQL agent",
          "MCP server",
          "answers with SQL and rows"
        ],
        [
          "MCP server",
          "AI agents",
          "nine read-only tools"
        ]
      ]
    },
    {
      "key": "beaver-no-hints-auto-ontology-against-a-general-agent",
      "title": "BEAVER, no hints: Auto Ontology against a general agent",
      "type": "chart",
      "source": "BEAVER leaderboard, read 8 November 2026",
      "caption": "Setting 0 (end to end, no annotations). Auto Ontology's entry was submitted by NVIDIA and lists no method details.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/beaver-no-hints-auto-ontology-against-a-general-agent.csv",
      "header": [
        "system",
        "Execution accuracy (%)"
      ],
      "rows": [
        [
          "ReFoRCE, Claude Opus 5.5",
          "14.0"
        ],
        [
          "NVIDIA Auto Ontology, Claude Opus 5.5",
          "12.4"
        ],
        [
          "ReFoRCE, Nemotron 3.5 Lightning",
          "9.8"
        ]
      ]
    },
    {
      "key": "appraising-the-evidence",
      "title": "Appraising the evidence",
      "type": "benchmarkTable",
      "source": "The papers and posts listed under Sources, read 3 and 8 November 2026",
      "caption": "Without and with curated business context, same model within each study.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/appraising-the-evidence.csv",
      "header": [
        "Study",
        "Who ran it",
        "Data",
        "Judged by",
        "Without",
        "With",
        "Strength"
      ],
      "rows": [
        [
          "data.world 2023",
          "Vendor (sells the layer)",
          "13 tables, 43 questions",
          "Execution match",
          "16.7%",
          "54.2%",
          "Weak: changed the query language, context and engine at once"
        ],
        [
          "dbt Labs 2026",
          "Vendor",
          "Same data, 11 questions x 20 runs",
          "Expected results, method not stated",
          "64.5%",
          "72.7%",
          "Medium: small set, newer models"
        ],
        [
          "Cube 2026",
          "Vendor",
          "Retail data, 99 questions",
          "LLM judge, lenient",
          "46.5%",
          "68.7%",
          "Medium: the layer was a 4 KB Markdown file"
        ],
        [
          "Snowflake 2025",
          "Vendor",
          "BIRD, 392 questions",
          "LLM judge",
          "57%",
          "78%",
          "Medium: vendor's own product"
        ],
        [
          "GSK 2024",
          "Self-graded",
          "500+ tables, 60 logged questions",
          "Domain expert",
          "8.3%",
          "78.3%",
          "Medium: real data, small set"
        ],
        [
          "LinkedIn 2025",
          "Self-graded",
          "Millions of tables, 133 questions",
          "LLM judge and experts",
          "9%",
          "48%",
          "Medium: real data, \"correct or close\""
        ],
        [
          "BIRD 2023",
          "Independent",
          "95 databases, one-sentence hints",
          "Execution match",
          "34.9%",
          "54.9%",
          "Strong"
        ],
        [
          "BEAVER 2026",
          "Independent",
          "Real warehouses, all oracle hints",
          "Execution match",
          "9.5%",
          "25.9%",
          "Strong: average of seven models"
        ],
        [
          "EntSQL 2026",
          "Independent",
          "Five business domains",
          "Execution match",
          "6.8%",
          "21.4%",
          "Strong"
        ]
      ]
    },
    {
      "key": "accuracy-without-and-with-curated-context-by-who-ran-the-study",
      "title": "Accuracy without and with curated context, by who ran the study",
      "type": "chart",
      "source": "Studies as in the table above",
      "caption": "The biggest jumps come from vendors and self-graded studies; independent benchmarks show about 15 to 20 points.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/accuracy-without-and-with-curated-context-by-who-ran-the-study.csv",
      "header": [
        "study",
        "Without context",
        "With context"
      ],
      "rows": [
        [
          "data.world 2023 (vendor)",
          "16.7",
          "54.2"
        ],
        [
          "dbt Labs 2026 (vendor)",
          "64.5",
          "72.7"
        ],
        [
          "Cube 2026, Sonnet 4.6 (vendor)",
          "46.5",
          "68.7"
        ],
        [
          "Snowflake 2025 (vendor)",
          "57",
          "78"
        ],
        [
          "GSK 2024 (self-graded)",
          "8.3",
          "78.3"
        ],
        [
          "LinkedIn 2025 (self-graded)",
          "9",
          "48"
        ],
        [
          "BIRD 2023, GPT-4 (independent)",
          "34.9",
          "54.9"
        ],
        [
          "BEAVER 2026 (independent)",
          "9.5",
          "25.9"
        ],
        [
          "EntSQL 2026 (independent)",
          "6.8",
          "21.4"
        ]
      ]
    },
    {
      "key": "best-results-on-text-to-sql-benchmarks",
      "title": "Best results on text-to-SQL benchmarks",
      "type": "benchmarkTable",
      "source": "BIRD and BEAVER leaderboards read 8 November 2026; Spider 2.0 leaderboard, BIRD-Interact and EntSQL papers read 3 November 2026; label audit arXiv 2601.08778",
      "caption": "Human experts score 92.96% on BIRD with hints. Annotation errors affect 52.8% of BIRD Mini-Dev and 62.8% of the checked Spider 2.0-Snow examples, so treat the top scores with care.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/best-results-on-text-to-sql-benchmarks.csv",
      "header": [
        "Benchmark",
        "What makes it realistic",
        "Best (%)"
      ],
      "rows": [
        [
          "BIRD, with hints",
          "95 databases, dirty values",
          "83.4"
        ],
        [
          "Spider 2.0-Snow",
          "Enterprise schemas on Snowflake",
          "96.7"
        ],
        [
          "Spider 2.0-Lite",
          "Three SQL dialects",
          "76.2"
        ],
        [
          "Spider 2.0-DBT",
          "A whole dbt project",
          "65.6"
        ],
        [
          "BEAVER, with annotations",
          "Real warehouses, schema and domain hints",
          "30.5"
        ],
        [
          "EntSQL, curated evidence",
          "Needs private business documents",
          "21.4"
        ],
        [
          "BIRD-Interact",
          "Must ask the user for missing knowledge",
          "17.0"
        ],
        [
          "BEAVER, no hints",
          "Real warehouses, real query logs",
          "14.0"
        ]
      ]
    },
    {
      "key": "auto-ontology-and-the-alternatives",
      "title": "Auto Ontology and the alternatives",
      "type": "benchmarkTable",
      "source": "Vendor documentation and pricing pages listed under Sources, read 3 and 8 November 2026",
      "caption": "Prices and status as of November 2026.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/auto-ontology-and-the-alternatives.csv",
      "header": [
        "Option",
        "How definitions are made",
        "Runs on",
        "Agent access",
        "Licence and price",
        "Status"
      ],
      "rows": [
        [
          "NVIDIA Auto Ontology",
          "LLM drafts from profiled metadata; experts certify",
          "Self-hosted; 10 SQL engines",
          "Read-only MCP, REST",
          "Apache-2.0; models and GPUs extra",
          "1.2.0, 6 Nov 2026"
        ],
        [
          "dbt Semantic Layer (MetricFlow)",
          "Hand-written YAML, AI-assisted; SQL compiled deterministically",
          "Your warehouse; hosted API on dbt platform",
          "dbt MCP server",
          "MetricFlow Apache-2.0; hosted from $100 per user per month",
          "Mature"
        ],
        [
          "Cube",
          "Data model as code, AI-assisted",
          "Self-hosted Core or Cube Cloud",
          "Hosted MCP, can edit the model",
          "Core open source; Cloud from $40 per developer per month",
          "Mature"
        ],
        [
          "Snowflake semantic views",
          "SQL, YAML or generated by Autopilot",
          "Snowflake only",
          "Managed MCP, Cortex Agents",
          "Snowflake consumption",
          "Available; Autopilot generally available"
        ],
        [
          "Databricks metric views and Genie Ontology",
          "YAML or UI, plus inferred context",
          "Databricks only",
          "Genie MCP",
          "Databricks consumption",
          "Ontology in preview"
        ],
        [
          "Google Looker (LookML)",
          "Hand-written LookML",
          "Looker",
          "Managed MCP",
          "Looker licence",
          "LookML mature; MCP in preview"
        ],
        [
          "Microsoft Fabric IQ and data agent",
          "Ontology generated from Power BI models",
          "Fabric only",
          "Fabric MCP servers",
          "Needs F2 capacity or Premium P1",
          "Ontology in preview"
        ],
        [
          "AtScale",
          "SML YAML with converters",
          "Over your warehouse",
          "MCP",
          "No public prices",
          "Long-standing vendor"
        ]
      ]
    },
    {
      "key": "accuracy-in-practice-generated-or-empty-layer-against-a-curated-one",
      "title": "Accuracy in practice: generated or empty layer against a curated one",
      "type": "benchmarkTable",
      "source": "Mastech, Microsoft ISE, Syren, Databricks and Anthropic posts, read 8 November 2026",
      "caption": "Most of these are vendors, partners or companies describing their own systems; none is an independent head-to-head test of products.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/accuracy-in-practice-generated-or-empty-layer-against-a-curated-one.csv",
      "header": [
        "Who",
        "Product",
        "Starting point",
        "After curation",
        "Caveats"
      ],
      "rows": [
        [
          "Mastech, for a client",
          "Snowflake Cortex Analyst",
          "63%",
          "77%",
          "30 questions; the starting view was generated by Cortex Code and gave silent wrong answers; the better one was built with a custom domain skill"
        ],
        [
          "Microsoft engineers (ISE)",
          "Databricks Genie",
          "9.5%",
          "88.5%",
          "Public benchmark; an empty space, then 69.2% with descriptions and 88.5% with a feedback loop that needed manual upkeep"
        ],
        [
          "Syren",
          "Databricks Genie",
          "53%",
          "80%",
          "15 questions; a space with a basic model, then remodelled; the later 100% came partly from changing the expected answers"
        ],
        [
          "Databricks worked example",
          "Databricks Genie",
          "0%",
          "100%",
          "Illustrative, unnamed team; 13 questions; 0 of 13 at first, 100% after five iterations"
        ],
        [
          "Anthropic, internal analytics",
          "Claude with a semantic layer and skills",
          "21%",
          "95%",
          "21% at most without the curated skills; about 95% with them, falling to about 65% in a month without upkeep; set size not disclosed"
        ]
      ]
    },
    {
      "key": "auto-ontology-defaults-and-pilot-settings",
      "title": "Auto Ontology defaults and pilot settings",
      "type": "benchmarkTable",
      "source": "model_config.py, .env.example, Helm values.yaml and docs, NVIDIA/auto-ontology, read 8 November 2026",
      "caption": "One stored connection per database: database row security does not follow the end user unless each connection is scoped.",
      "csv": "https://bardak.ai/data/nvidia-auto-ontology/auto-ontology-defaults-and-pilot-settings.csv",
      "header": [
        "Default",
        "What it means",
        "Pilot setting"
      ],
      "rows": [
        [
          "Hallucination checks (DB_PROBE_*) off",
          "Made-up values and joins can return wrong results instead of errors",
          "Turn all three on and accept the extra queries"
        ],
        [
          "An sk- or unknown API key falls back to Claude Opus 5.5 through an NVIDIA gateway",
          "Schema metadata, sample values and questions leave your network",
          "Point every model endpoint at an approved or self-hosted model"
        ],
        [
          "Vault off unless all four variables are set",
          "Database passwords stored as plain JSON in Postgres",
          "Configure Vault fully"
        ],
        [
          "Helm chart enables pgAdmin with admin@example.com / admin",
          "A database console with sample credentials on every node",
          "Disable pgAdmin; put an ingress with TLS in front"
        ],
        [
          "The backend trusts a user-id header; ports published on the host",
          "Anyone who reaches the port can act as any user",
          "Keep the services on an internal network"
        ],
        [
          "API tokens never expire by default and have no scopes",
          "A leaked token is a full user",
          "Set expiry; issue tokens per pilot user"
        ]
      ]
    }
  ]
}