{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/verify-claims-against-sources/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/verify-claims-against-sources/fixture.jsonl",
      "sha256": "3cf36a30dd46ea63cfb725b28e8ede51e7010b60d19f254ae92ef6ed67240ab8",
      "items": 100,
      "dev_items": 30,
      "split": "test",
      "source": "Original synthetic fixture: fictional observatory sources, generated drafts and generator-intended labels",
      "license": "CC0",
      "generators": []
    },
    "prompt_sha256": "1bf6a8e8b5bd58c2bbb78d0f6046200dc8c0c20e973cc551704bad88800a81aa",
    "recipe_sha256": "12f6306660c3763642b47b5df28b9692de9573379b8b27a11b0ab44074187019",
    "eval_json_sha256": "314514f3def2e7c806904798031b23f25c0f2cb70bf41d2835b2e7313d897762",
    "freeze_sha": "79122bd316d38cbb58c856459b85902c36d6515b",
    "repeats": 3,
    "seed": 421,
    "unit": "claim",
    "aggregation": "summed_counts",
    "headline": {
      "metric": "recall",
      "higher_is_better": true,
      "unit": "fraction"
    },
    "secondary": [
      {
        "id": "caught_errors",
        "metric": "count",
        "higher_is_better": true,
        "unit": "claim",
        "aggregation": "median"
      },
      {
        "id": "false_alarms",
        "metric": "count",
        "higher_is_better": false,
        "unit": "claim",
        "aggregation": "median"
      },
      {
        "id": "false_alarm_rate",
        "metric": "recall",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "quote_match_rate",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "label_accuracy",
        "metric": "accuracy",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "majority_label",
        "labels": [
          "supported",
          "contradicted",
          "not_in_sources",
          "invalid"
        ]
      },
      {
        "id": "invalid_output_rate",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "seeded_errors",
        "metric": "count",
        "higher_is_better": true,
        "unit": "claim",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 50640,
              "output": 5486,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "3d3cb380c16b74566b6b5a51b99b75c8eb1b1b3697d81736ae3a97cb2bdd86e6"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 50640,
              "output": 5497,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "c502877039021f4480c389df69e45b1a01c09fa3c645d2e3ba6884114333eabf"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 50640,
              "output": 5452,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "9cba2e97c9fe6003439d8ebfeec5f703ef1aaa75e0e990bf516401fc7b4ae2eb"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "low"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 64710,
              "output": 18710,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "89917ad7736b042414928c91e495a44e7da6dac3571e33e2690dff000b5f8fd2"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 64710,
              "output": 18746,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "bac35d82e15623914c4653527b1f9f2713f376541cc011343fea63270393d476"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 64710,
              "output": 19023,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "1e98ce6d760e0dec5c9f8e9e7bcd94da96c5d39fde93f5e183d70c1e31ca252f"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/verify-claims-against-sources/runs/2026-10-28T00-34-10-601840Z/raw.jsonl.gz",
      "sha256": "49356bbdd89757eae9a10c3b0eb43c4fdb7f1b997cd3db6d8682862d98974c72"
    }
  }
}