{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/transaction-categorization/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/transaction-categorization/fixture.jsonl",
      "sha256": "2279e83f8bd4c74d6124193a95bdfde5a44388b52b5881d385b4eb217e14418a",
      "items": 50,
      "dev_items": 10,
      "split": "test",
      "generators": [],
      "license": "CC BY 4.0",
      "source": "Synthetic bank transactions generated by scripts/evals/transaction-categorization/generate_fixture.py; labels are generator-intended and disclosed"
    },
    "prompt_sha256": "4dc8d6fb5f1ce67c6f34a93f70062fc316f19ae22032cd2063e708bc54885fe4",
    "recipe_sha256": "480396c2b83e61be0bea7a595678c4a74237ced22c4203277ab4c49b290eb8fd",
    "eval_json_sha256": "34f8a296cfa299ca2ca866bb7016dd62f3fe5c202b7c66e4452e38e23c269487",
    "freeze_sha": "5f0f00c52d4dc20c406c084d5358bc4644e2fdd8",
    "repeats": 3,
    "seed": 42,
    "unit": "batch of 10 transactions",
    "aggregation": "median",
    "headline": {
      "higher_is_better": true,
      "metric": "mean",
      "unit": "transaction accuracy"
    },
    "secondary": [
      {
        "aggregation": "median",
        "higher_is_better": true,
        "id": "oneshot_accuracy",
        "metric": "mean",
        "unit": "transaction accuracy"
      },
      {
        "aggregation": "median",
        "higher_is_better": true,
        "id": "batch_macro_f1",
        "metric": "mean",
        "unit": "fraction"
      },
      {
        "aggregation": "median",
        "higher_is_better": false,
        "id": "invented_account_rate",
        "metric": "mean",
        "unit": "fraction"
      },
      {
        "aggregation": "summed_counts",
        "higher_is_better": true,
        "id": "needs_review_precision",
        "metric": "precision",
        "unit": "fraction"
      },
      {
        "aggregation": "summed_counts",
        "higher_is_better": true,
        "id": "needs_review_recall",
        "metric": "recall",
        "unit": "fraction"
      },
      {
        "aggregation": "median",
        "higher_is_better": true,
        "id": "tool_call_validity",
        "metric": "mean",
        "unit": "fraction"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 1350,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 246152,
              "output": 40833,
              "cached": 2464,
              "reasoning": 1948
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "76b28ba96ed62db9a1d9bd84ac42b1eba72f61cf2ab04b6d68f37b57255b0b2e"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 246152,
              "output": 41106,
              "cached": 144214,
              "reasoning": 2152
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "866d3ea9ca42ba42c7f3fac6b57968994ffbf65583634613d9dbb2ad1a744f14"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 246152,
              "output": 41080,
              "cached": 144214,
              "reasoning": 2001
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "3019967c6d3fcc85ad525b91c1d00df2cff7e05e49890d4a96dd8249c0ff90b4"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 1350,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 324600,
              "output": 131483,
              "cached": 100589,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 1,
            "response_ids_sha": "443dcca7c00f5f86ea3172c23069127f5fee3f961a83612ad7d10f227ec2779e"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 327074,
              "output": 135257,
              "cached": 115607,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "3603ffe5e86685dbd6d7b557a935641aaf509342608e24512fd53e715fc5812d"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 329608,
              "output": 134735,
              "cached": 92440,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 2,
            "response_ids_sha": "e1440e5620e7fdc93030059a79b2fc4affeb0971a90f14a8708d7e673afda56c"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/transaction-categorization/runs/2026-10-28T00-55-53-921024Z/raw.jsonl.gz",
      "sha256": "bc41c1b9f4cd748f066f87206905a77f1aebb041265f937f44e1731fbf7e1805"
    }
  }
}