{
  "unit": "pair",
  "aggregation": "majority_label",
  "headline": {
    "metric": "kappa",
    "higher_is_better": true,
    "unit": "agreement",
    "labels": [
      "A",
      "B",
      "equal",
      "invalid"
    ]
  },
  "secondary": [
    {
      "id": "agreement",
      "metric": "accuracy",
      "higher_is_better": true,
      "unit": "pair",
      "aggregation": "majority_label",
      "labels": [
        "A",
        "B",
        "equal",
        "invalid"
      ]
    },
    {
      "id": "position_consistency",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "median"
    },
    {
      "id": "valid_output",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "median"
    }
  ],
  "repeats": 5,
  "seed": 20261027,
  "verdict_rule": {
    "bootstrap": "paired-items",
    "resamples": 10000,
    "confidence": 0.95,
    "prng": "mulberry32",
    "quantile": "type-7",
    "range": "absolute-difference-exceeds-wider-run-range",
    "label_tie": "json-lexicographic",
    "zero_division": 0,
    "degenerate_kappa": 1,
    "rounding": "half-away-from-zero",
    "decimals": 4
  },
  "record_shape": [
    "gold",
    "prediction"
  ],
  "prompt": "You are grading two anonymous answers to one question for a person who needs to pick one answer. Judge which answer better serves the request: correctness, completeness and fit to the instructions. Judge only the two answers as written; do not rewrite either. Answer A and answer B are equally weighted candidates; neither has an advantage. If both answers are equally good or equally flawed, record choice equal instead of forcing a winner. Record a reason of at most two sentences describing what separates the answers. Do not identify or guess either model. The question and both answers arrive as JSON. Consider both answers in full before choosing.",
  "fixture_sha256": "4ba4954e887587ee9d036490ca79a0ba4e8ef2a88a7aa57d596dfff0de6f1f50",
  "recipe_sha256": "832cbbdb5d51d555a5a9981beb7d6c9d37c1e50cce8483e530ce8df36bffd89e",
  "checker_sha256": "c40f83acbfe1e0a48b554cac3dc23093fb10bbb150ce536a77edcdd38220bd77",
  "runner_sha256": "2565828a1843a1b8fd95a587165dd6919a23e7a6dcdc91161dc9747e085027b9",
  "max_tokens": 512,
  "reasoning_effort": "medium",
  "tool_choice": {
    "type": "function",
    "function": {
      "name": "grade"
    }
  },
  "tool_steps": [
    {
      "type": "function",
      "function": {
        "name": "grade"
      }
    }
  ],
  "planning": {
    "paired_difference_sd": 0.35
  },
  "fixture_source": {
    "source": "lmsys/mt_bench_human_judgments human split, turn 1, two annotators per pair",
    "license": "CC BY 4.0",
    "generators": [
      "source dataset models, blinded to A/B"
    ]
  }
}