{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/support-ticket-triage/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/support-ticket-triage/fixture.jsonl",
      "sha256": "01acfa159c41cdb5c1f3cf091c3b2103663bbb272273c9ba37cf754ee4ecd938",
      "items": 100,
      "dev_items": 30,
      "split": "test",
      "source": "130 synthetic tickets written for this evaluation",
      "license": "CC BY 4.0",
      "generators": [
        "Pareto coding agent; deterministic Python templates"
      ]
    },
    "prompt_sha256": "8dacc68bdffc8ba681fe8d2982c466e2ccfd42438d3796576018e626c2d30497",
    "recipe_sha256": "290d03c422efaa056f1ab534066351cb236e7e174f5ad9f2132e7f25f17335f3",
    "eval_json_sha256": "e6b109684fea2af30926d9f6aede7900f0d0a7b43ad74a40f67b50aa5bf60382",
    "freeze_sha": "c6daeac91cf469b9e9913a219382823c2c264b71",
    "repeats": 3,
    "seed": 20261027,
    "unit": "ticket",
    "aggregation": "majority_label",
    "headline": {
      "metric": "macro_f1",
      "higher_is_better": true,
      "unit": "fraction",
      "labels": [
        "billing",
        "access",
        "bug",
        "feature",
        "unknown"
      ]
    },
    "secondary": [
      {
        "id": "category_accuracy",
        "metric": "accuracy",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "majority_label",
        "labels": [
          "billing",
          "access",
          "bug",
          "feature",
          "unknown"
        ]
      },
      {
        "id": "priority_accuracy",
        "metric": "accuracy",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "majority_label",
        "labels": [
          "P1",
          "P2",
          "P3"
        ]
      },
      {
        "id": "priority_f1",
        "metric": "macro_f1",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "majority_label",
        "labels": [
          "P1",
          "P2",
          "P3"
        ]
      },
      {
        "id": "team_accuracy",
        "metric": "accuracy",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "majority_label",
        "labels": [
          "finance",
          "identity",
          "engineering",
          "product",
          "support"
        ]
      },
      {
        "id": "team_f1",
        "metric": "macro_f1",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "majority_label",
        "labels": [
          "finance",
          "identity",
          "engineering",
          "product",
          "support"
        ]
      },
      {
        "id": "p1_recall",
        "metric": "recall",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "priority_off_by_one",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "invalid_enum",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "exact_match",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "confidence_low",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "confidence_medium",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "confidence_high",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "seed_template_accuracy",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "ambiguous_accuracy",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "long_thread_accuracy",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "non_english_accuracy",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "no_detail_accuracy",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 2048,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 38193,
              "output": 6400,
              "cached": 0,
              "reasoning": 205
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "743544e386a32fdb7cc9af77148bd5bd1b7e977faef505cf02e09f9ffea47dee"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 38193,
              "output": 6353,
              "cached": 0,
              "reasoning": 142
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "fd6d3254a14ba495e4fa9905d7bd41d6d26a8fdae5bf2a1940f0f1a82f7300ee"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 38193,
              "output": 6397,
              "cached": 0,
              "reasoning": 199
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "d5af03f7df54ea506c46f2a16bea160cf43db31bcf7651874d5735d7640a4d94"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "low"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 2048,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 54195,
              "output": 38175,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "b32d05a8256ba568d6e90651044ba2daf943b56205028115c4e141d9b973140e"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 54171,
              "output": 36997,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "a709eb5e77769fb322bdff519f901f99419020563887081d326ba96c017e61b0"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 54728,
              "output": 36554,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "2e4e67476c03bf643bf885145cf642bf1ce2f3e9b31e552b1df92f74df5d466c"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/support-ticket-triage/runs/2026-10-28T00-10-50-935281Z/raw.jsonl.gz",
      "sha256": "e6187905e7f5daa29b634bab5e4c9f14e2c035efcccdfc82b919c85abfc2e2c1"
    }
  }
}