{
  "post": {
    "title": "How to evaluate GraphRAG honestly: golden sets, metrics and judge bias",
    "url": "https://bardak.ai/posts/graphrag-evaluation",
    "updated": "2026-11-17T07:00:25.356Z"
  },
  "citation": "How to evaluate GraphRAG honestly: golden sets, metrics and judge bias, Ivan Bardak, bardak.ai, https://bardak.ai/posts/graphrag-evaluation",
  "datasets": [
    {
      "key": "lightrag-win-rate-as-reported-and-under-a-controlled-protocol",
      "title": "LightRAG win rate: as reported and under a controlled protocol",
      "type": "chart",
      "source": "Zeng et al., How Significant Are the Real Performance Gains? An Unbiased Evaluation Framework for GraphRAG (arXiv 2506.06331, 2025)",
      "caption": "Agriculture dataset. Under the controlled protocol plain NaiveRAG beat LightRAG, and more than a fifth of comparisons were ties in most settings.",
      "csv": "https://bardak.ai/data/graphrag-evaluation/lightrag-win-rate-as-reported-and-under-a-controlled-protocol.csv",
      "header": [
        "comparison",
        "As reported",
        "Controlled protocol"
      ],
      "rows": [
        [
          "LightRAG vs NaiveRAG",
          "66.7",
          "39.1"
        ],
        [
          "LightRAG vs MGRAG",
          "56.4",
          "32.3"
        ]
      ]
    },
    {
      "key": "position-consistency-of-llm-judges",
      "title": "Position consistency of LLM judges",
      "type": "benchmarkTable",
      "source": "Shi et al., Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge (AACL 2025), Table 2",
      "caption": "Share of comparisons with the same verdict after the answers are swapped (1.0 means fully consistent). Consistency depends on the judge and the task, and falls when the two answers are close in quality, which is exactly the GraphRAG versus RAG situation.",
      "csv": "https://bardak.ai/data/graphrag-evaluation/position-consistency-of-llm-judges.csv",
      "header": [
        "Judge",
        "MT-Bench",
        "DevBench"
      ],
      "rows": [
        [
          "GPT-4",
          "0.82",
          "0.83"
        ],
        [
          "Llama-3.3-70B",
          "0.80",
          "0.89"
        ],
        [
          "GPT-4o",
          "0.76",
          "0.80"
        ],
        [
          "Claude-3.5-Sonnet",
          "0.82",
          "0.76"
        ],
        [
          "Gemini-1.5-pro",
          "0.62",
          "0.84"
        ],
        [
          "Llama-3.1-8B",
          "0.69",
          "0.47"
        ],
        [
          "Claude-3-Haiku",
          "0.57",
          "0.23"
        ]
      ]
    },
    {
      "key": "rag-and-graphrag-scored-against-reference-answers",
      "title": "RAG and GraphRAG scored against reference answers",
      "type": "benchmarkTable",
      "source": "Han et al., RAG vs. GraphRAG: A Systematic Evaluation and Key Insights (arXiv 2502.11371, v3)",
      "caption": "Plain RAG leads on single-fact questions, graph methods on multi-hop ones, by narrow margins. Only about two thirds of the answer entities existed in the graphs built from the documents.",
      "csv": "https://bardak.ai/data/graphrag-evaluation/rag-and-graphrag-scored-against-reference-answers.csv",
      "header": [
        "Method",
        "Natural Questions F1",
        "HotpotQA F1",
        "MultiHop-RAG accuracy"
      ],
      "rows": [
        [
          "Plain RAG",
          "64.78",
          "60.04",
          "67.02"
        ],
        [
          "Community-GraphRAG (local)",
          "63.01",
          "61.66",
          "69.01"
        ],
        [
          "HippoRAG 2",
          "61.03",
          "63.01",
          "70.27"
        ],
        [
          "Knowledge-graph triplets only",
          "34.28",
          "",
          ""
        ]
      ]
    },
    {
      "key": "which-metric-measures-what",
      "title": "Which metric measures what",
      "type": "benchmarkTable",
      "source": "Rajpurkar et al. 2016 (SQuAD); Thakur et al. 2021 (BEIR); Es et al. 2023 (RAGAS)",
      "caption": "Reference-free metrics are useful for monitoring, but they cannot tell you whether an answer is correct.",
      "csv": "https://bardak.ai/data/graphrag-evaluation/which-metric-measures-what.csv",
      "header": [
        "Metric",
        "What it measures",
        "Needs reference answers?"
      ],
      "rows": [
        [
          "Exact match and F1",
          "Whether the answer matches a reference answer exactly, or how many words overlap",
          "Yes"
        ],
        [
          "Claim-level recall",
          "Share of the facts in the reference answer that the answer contains",
          "Yes"
        ],
        [
          "Recall@k",
          "Share of the relevant passages that appear in the top k retrieved",
          "Labelled passages"
        ],
        [
          "nDCG@10",
          "Retrieval quality that rewards putting the best passages first",
          "Labelled passages"
        ],
        [
          "Faithfulness (RAGAS)",
          "Share of the answer's statements supported by the retrieved context",
          "No"
        ],
        [
          "Answer relevance (RAGAS)",
          "How closely the answer addresses the question; ignores whether it is true",
          "No"
        ],
        [
          "Pairwise LLM judgement",
          "Which of two answers a model prefers",
          "No, which is why it is weakest"
        ]
      ]
    },
    {
      "key": "appraising-the-evidence",
      "title": "Appraising the evidence",
      "type": "benchmarkTable",
      "source": "",
      "caption": "",
      "csv": "https://bardak.ai/data/graphrag-evaluation/appraising-the-evidence.csv",
      "header": [
        "Evidence",
        "Strength",
        "Main limitation"
      ],
      "rows": [
        [
          "Studies of judge bias (Zheng, Shi, Wang)",
          "Large, repeated, consistent across judges",
          "General chat tasks, not retrieval; judges improve over time"
        ],
        [
          "GraphRAG evaluation studies (Han, Zeng)",
          "Independent re-evaluations with controls",
          "Specific datasets and methods; Zeng et al. is a single preprint"
        ],
        [
          "Benchmarks (Xiao, Xiang)",
          "Reference answers, many methods",
          "Their corpora, not yours"
        ],
        [
          "Statistical guidance (Miller, Bowyer)",
          "Sound methods with worked examples",
          "Assumptions about variance that you should check on your data"
        ]
      ]
    }
  ]
}