{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/invoice-data-extraction/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/invoice-data-extraction/fixture.jsonl",
      "sha256": "a28cb8a9eee11fb196185071b5001b0593e789d7ee4198e9c16a2a4e4702385e",
      "items": 40,
      "dev_items": 10,
      "split": "test",
      "source": "Digital invoice PDFs we generated with reportlab 5.1.0 and converted to text with markitdown 0.1.9; 8 layouts, 4 currencies, 2 credit notes, 6 planted total errors in the test split and 2 in dev",
      "license": "CC BY 4.0",
      "generators": []
    },
    "prompt_sha256": "d7bac683fca33fc2e957bab721bf556b76858b72f50241581b78415921c37119",
    "recipe_sha256": "8d582fa9010236a9711ee38f2a6593ad708bc3559c85614507a785fe9537ed29",
    "eval_json_sha256": "f1ce9703f3ca0950b5882627a9018164f53d5bda7197ec27a91ffe97339756a6",
    "freeze_sha": "749cd69135082aa6eab34be26d5a42c1be3d2569",
    "repeats": 3,
    "seed": 42,
    "unit": "invoice",
    "aggregation": "median",
    "headline": {
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of 12 invoice fields exact per record"
    },
    "secondary": [
      {
        "id": "line_item_f1",
        "metric": "f1",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "schema_validity",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "planted_errors_caught",
        "metric": "count",
        "higher_is_better": true,
        "unit": "invoices of 6 planted (or 34 clean) per run",
        "aggregation": "median"
      },
      {
        "id": "false_flags",
        "metric": "count",
        "higher_is_better": false,
        "unit": "invoices of 6 planted (or 34 clean) per run",
        "aggregation": "median"
      },
      {
        "id": "invoice_id_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "document_type_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "vendor_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "email_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "invoice_date_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "due_date_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "currency_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "subtotal_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "tax_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "total_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "bank_code_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      },
      {
        "id": "purchase_order_exact",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction of records",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 21236,
              "output": 6780,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "985e1731baaed0a532379fe4c925427ff54329c85823a333180fb67f1404ff5c"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 21236,
              "output": 6769,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "3b83de25a6720a713aa25b7ac5b7bc92b00d8c117ded451730b8382e87ae6b64"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 21236,
              "output": 6776,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "d076eb9b6ce946bfcbf01d6e03609e61f293fb8cdb47652d5faf71b1c9c2f846"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "low"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 33137,
              "output": 20837,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "60207982bf740e78e729bd084e922474e63ef3365f2ea316231f2415967e839e"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 33137,
              "output": 19959,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "ab3ce45611a0f07ed209a03029b02908648fe5d49917a63e60c0d1546f04b7dc"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 33137,
              "output": 19640,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "b19bd48d73a86669629c3bb6fbca98e229c183d669ee94440c2852113d7557cd"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {
      "pydantic-ai": {
        "runs": [
          {
            "repeat": 1,
            "split": "dev",
            "result": "pass"
          },
          {
            "repeat": 2,
            "split": "dev",
            "result": "pass"
          },
          {
            "repeat": 3,
            "split": "dev",
            "result": "pass"
          }
        ],
        "checker": "check.py",
        "files_sha256": {
          "recipe.py": "2876d4398e9a6dcedb5a99b461800585734495bb29c604acc556386d1fa1c685",
          "check.py": "6786258c322de280081a2b4ac7de936e6d767708f51778327a7a54993913b7fd",
          "prompt.py": "39536c04424625486c242b7450be130e16670fbfe75e17356780809a763e8559"
        }
      }
    },
    "raw": {
      "path": "data/use-case-evals/invoice-data-extraction/runs/2026-10-28T00-10-20-775558Z/raw.jsonl.gz",
      "sha256": "61f4df08a2c77e5d780f8cf8f04d87c17f5566412e8116d18e5ba68fb3587df6"
    }
  }
}