{
  "unit": "invoice",
  "aggregation": "median",
  "headline": {
    "metric": "mean",
    "higher_is_better": true,
    "unit": "fraction of 12 invoice fields exact per record"
  },
  "secondary": [
    {
      "id": "line_item_f1",
      "metric": "f1",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "schema_validity",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "planted_errors_caught",
      "metric": "count",
      "higher_is_better": true,
      "unit": "invoices of 6 planted (or 34 clean) per run",
      "aggregation": "median"
    },
    {
      "id": "false_flags",
      "metric": "count",
      "higher_is_better": false,
      "unit": "invoices of 6 planted (or 34 clean) per run",
      "aggregation": "median"
    },
    {
      "id": "invoice_id_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "document_type_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "vendor_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "email_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "invoice_date_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "due_date_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "currency_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "subtotal_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "tax_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "total_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "bank_code_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    },
    {
      "id": "purchase_order_exact",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction of records",
      "aggregation": "median"
    }
  ],
  "repeats": 3,
  "seed": 42,
  "verdict_rule": {
    "bootstrap": "paired-items",
    "resamples": 10000,
    "confidence": 0.95,
    "prng": "mulberry32",
    "quantile": "type-7",
    "range": "absolute-difference-exceeds-wider-run-range",
    "label_tie": "json-lexicographic",
    "zero_division": 0,
    "degenerate_kappa": 1,
    "rounding": "half-away-from-zero",
    "decimals": 4
  },
  "record_shape": [
    "score"
  ],
  "prompt": "You extract printed invoice and credit-note fields into one record tool call. One document per call. Copy every value exactly as the document prints it, including currency signs, minus marks and leading zeros. Do not compute, convert or correct any amount; if the document's printed total disagrees with subtotal plus tax, print the document's total anyway and say so in note. invoice_id, vendor, email, invoice_date, due_date, currency, subtotal, tax, total, bank_code and purchase_order come straight from the printed text. document_type is invoice or credit_note. invoice_date and due_date keep the document's YYYY-MM-DD form. subtotal, tax, total, quantity, unit_price and amount are plain decimal numbers without currency symbols or thousands separators. line_items repeats the printed line-item table rows in document order, one row object each with description, quantity, unit_price and amount as printed. purchase_order is null when the document does not show one. note is a short string: mention a missing field, a total that does not add up, or an empty string when nothing is unusual. Call the tool exactly once.",
  "fixture_sha256": "a28cb8a9eee11fb196185071b5001b0593e789d7ee4198e9c16a2a4e4702385e",
  "recipe_sha256": "8d582fa9010236a9711ee38f2a6593ad708bc3559c85614507a785fe9537ed29",
  "checker_sha256": "f2e2da02a8e76df38ac2f040394eba2bbf1240e43c4bf6d64047100ad83315ea",
  "runner_sha256": "2ef5bd134dfaa1491d139676b7d2e5da4b8e29a3a3edd8ccbdb98d3076118e7e",
  "max_tokens": 4096,
  "reasoning_effort": "low",
  "tool_choice": {
    "type": "function",
    "function": {
      "name": "record"
    }
  },
  "tuning_repeats": 1,
  "fixture_source": {
    "source": "Digital invoice PDFs we generated with reportlab 5.1.0 and converted to text with markitdown 0.1.9; 8 layouts, 4 currencies, 2 credit notes, 6 planted total errors in the test split and 2 in dev",
    "license": "CC BY 4.0",
    "generators": []
  },
  "planning": {
    "paired_difference_sd": 0.08
  },
  "variants": {
    "pydantic-ai": {
      "runs": [
        {
          "repeat": 1,
          "split": "dev",
          "result": "pass"
        },
        {
          "repeat": 2,
          "split": "dev",
          "result": "pass"
        },
        {
          "repeat": 3,
          "split": "dev",
          "result": "pass"
        }
      ],
      "checker": "check.py",
      "files_sha256": {
        "recipe.py": "2876d4398e9a6dcedb5a99b461800585734495bb29c604acc556386d1fa1c685",
        "check.py": "6786258c322de280081a2b4ac7de936e6d767708f51778327a7a54993913b7fd",
        "prompt.py": "39536c04424625486c242b7450be130e16670fbfe75e17356780809a763e8559"
      }
    }
  }
}