{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/research-paper-data-extraction/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/research-paper-data-extraction/fixture.jsonl",
      "sha256": "faef14282b0fcf7b234ca55ba3febeec414f088320f2ac2256caab223926d6d2",
      "items": 20,
      "dev_items": 4,
      "split": "test",
      "source": "Original synthetic trial excerpts; generator-intended labels, automated spot-check; no human adjudication",
      "license": "CC0",
      "generators": []
    },
    "prompt_sha256": "ad66e9e44c7a7e80c8c6254c2a1a7f01ee3fe9ab3ace79049acfb959762acc3f",
    "recipe_sha256": "37f9d8ae057564e28df9300fa6763126798263dc2fa490fe743b706ca6749e06",
    "eval_json_sha256": "01bacb6f6e60f029f59dad980b33dfd324a3996eb27e0ddbfd451e7fc11b3804",
    "freeze_sha": "13eae612a54b3a4e438beaa5ba58e57c8745209b",
    "repeats": 3,
    "seed": 20261027,
    "unit": "paper",
    "aggregation": "median",
    "headline": {
      "metric": "mean",
      "higher_is_better": true,
      "unit": "field accuracy"
    },
    "secondary": [
      {
        "id": "quote_match",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "review_load",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "schema_valid",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 12878,
              "output": 7337,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "0ece14addaa1fe21ecec62855ef40568573b5ceb85987fa097ef9455e34645e2"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 12878,
              "output": 7336,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "86b5c1931961830b57faf0a393625207dd605da088f4bfe2d7f4c7aa179a325f"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 12878,
              "output": 7335,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "7ab5bc4cf55fde33aa55f73b58b4f4c3bfd54b237eb4ef616f8155cdb5a1cffb"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 27305,
              "output": 18251,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "5d436d2135e10c15482055f73a98fdd9c657ab61707694d7d9713283d3de0191"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 27305,
              "output": 17787,
              "cached": 11660,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "761d19517139f59d516b02f18f39e2cf1dd857703bcbd161c06e7e4247bbede4"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 27305,
              "output": 17272,
              "cached": 11660,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "11d43fa78594029231e20293e9c9f2e601d7362d24a5a3437997ee5f57c9fa54"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/research-paper-data-extraction/runs/2026-10-28T00-10-31-261064Z/raw.jsonl.gz",
      "sha256": "5e404abc0eba0989cf7180d5823a52eeed729b6a7fee4d7932a713bebc231696"
    }
  }
}