{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/llm-as-a-judge/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/llm-as-a-judge/fixture.jsonl",
      "sha256": "4ba4954e887587ee9d036490ca79a0ba4e8ef2a88a7aa57d596dfff0de6f1f50",
      "items": 100,
      "dev_items": 20,
      "split": "test",
      "source": "lmsys/mt_bench_human_judgments human split, turn 1, two annotators per pair",
      "license": "CC BY 4.0",
      "generators": [
        "source dataset models, blinded to A/B"
      ]
    },
    "prompt_sha256": "79e9597efed81701db93c6c8ab72fc2724a9d27d4b93d242e0004aeafb09e308",
    "recipe_sha256": "832cbbdb5d51d555a5a9981beb7d6c9d37c1e50cce8483e530ce8df36bffd89e",
    "eval_json_sha256": "eb21aa1fc3b5f1974725cde0c6efa2de85ffa663f2c026acf6700b8c695bb7d5",
    "freeze_sha": "e1c038ac8ee5b30da7d7d61070ae552219f7a8ef",
    "repeats": 5,
    "seed": 20261027,
    "unit": "pair",
    "aggregation": "majority_label",
    "headline": {
      "metric": "kappa",
      "higher_is_better": true,
      "unit": "agreement",
      "labels": [
        "A",
        "B",
        "equal",
        "invalid"
      ]
    },
    "secondary": [
      {
        "id": "agreement",
        "metric": "accuracy",
        "higher_is_better": true,
        "unit": "pair",
        "aggregation": "majority_label",
        "labels": [
          "A",
          "B",
          "equal",
          "invalid"
        ]
      },
      {
        "id": "position_consistency",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "valid_output",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 512,
        "runs": [
          {
            "repeat": 1,
            "requests": 200,
            "tokens": {
              "input": 133874,
              "output": 18971,
              "cached": 0,
              "reasoning": 5200
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "87d859c5e5f46fabbf203a977a24fa50e096067b5749318db21adbdf5106881b"
          },
          {
            "repeat": 2,
            "requests": 200,
            "tokens": {
              "input": 133874,
              "output": 18978,
              "cached": 15863,
              "reasoning": 5111
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "f22124a7e6038a90cf8ba614f130550feb7aad37cd7e4d31458d6fac8a393bc4"
          },
          {
            "repeat": 3,
            "requests": 200,
            "tokens": {
              "input": 133874,
              "output": 19321,
              "cached": 15863,
              "reasoning": 5498
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "07b1e089bf14f877a05b4d44f94be9a3a5729a8fbfc75fbb75cf8ac7515a7ecc"
          },
          {
            "repeat": 4,
            "requests": 200,
            "tokens": {
              "input": 133874,
              "output": 19261,
              "cached": 15863,
              "reasoning": 5486
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "f3a48e76f07f98f26f3fea425b8036ed1a1c2659dd117a38bfb3a05e128c3b6c"
          },
          {
            "repeat": 5,
            "requests": 200,
            "tokens": {
              "input": 133874,
              "output": 19352,
              "cached": 15863,
              "reasoning": 5502
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "18a6665fee35f23a986459d4636289938d066d16bb3413dae2322bee1406c23f"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 512,
        "runs": [
          {
            "repeat": 1,
            "requests": 200,
            "tokens": {
              "input": 157786,
              "output": 15159,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 1,
            "retries": 0,
            "truncations": 1,
            "response_ids_sha": "ded7ff92b7ec7b96a705240ef897c16169ada6ca94e4cf29ca78b1fd3b35f44c"
          },
          {
            "repeat": 2,
            "requests": 200,
            "tokens": {
              "input": 157786,
              "output": 15718,
              "cached": 11774,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 2,
            "retries": 0,
            "truncations": 2,
            "response_ids_sha": "ad276b1f17ff773ce40c1ac8ecf212771d805c18c45d8e681b9c0dd02a7a6024"
          },
          {
            "repeat": 3,
            "requests": 200,
            "tokens": {
              "input": 157786,
              "output": 15149,
              "cached": 11191,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "30baaf9dc4773a31c61cfd3139e1f788ebfe47b5345c5b0d2963ef00d63eef07"
          },
          {
            "repeat": 4,
            "requests": 200,
            "tokens": {
              "input": 157786,
              "output": 14662,
              "cached": 11774,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "7f0735897e7c59d5a2d70c2ed4e0bf775ae6479b51c0f55778bc79e41ebb0be6"
          },
          {
            "repeat": 5,
            "requests": 200,
            "tokens": {
              "input": 157786,
              "output": 15059,
              "cached": 11774,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "7b9ce8924bf8e4398e05dcbb62c978ae03953869d6ada88e003a8aa347a65cab"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/llm-as-a-judge/runs/2026-10-28T00-12-11-323493Z/raw.jsonl.gz",
      "sha256": "d5a8a69c1c2b829669467649702f16bfaf1555d40f410f9874bb19170caefe65"
    }
  }
}