{
  "unit": "ticket",
  "aggregation": "majority_label",
  "headline": {
    "metric": "macro_f1",
    "higher_is_better": true,
    "unit": "fraction",
    "labels": [
      "billing",
      "access",
      "bug",
      "feature",
      "unknown"
    ]
  },
  "secondary": [
    {
      "id": "category_accuracy",
      "metric": "accuracy",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "majority_label",
      "labels": [
        "billing",
        "access",
        "bug",
        "feature",
        "unknown"
      ]
    },
    {
      "id": "priority_accuracy",
      "metric": "accuracy",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "majority_label",
      "labels": [
        "P1",
        "P2",
        "P3"
      ]
    },
    {
      "id": "priority_f1",
      "metric": "macro_f1",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "majority_label",
      "labels": [
        "P1",
        "P2",
        "P3"
      ]
    },
    {
      "id": "team_accuracy",
      "metric": "accuracy",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "majority_label",
      "labels": [
        "finance",
        "identity",
        "engineering",
        "product",
        "support"
      ]
    },
    {
      "id": "team_f1",
      "metric": "macro_f1",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "majority_label",
      "labels": [
        "finance",
        "identity",
        "engineering",
        "product",
        "support"
      ]
    },
    {
      "id": "p1_recall",
      "metric": "recall",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "priority_off_by_one",
      "metric": "mean",
      "higher_is_better": false,
      "unit": "fraction",
      "aggregation": "median"
    },
    {
      "id": "invalid_enum",
      "metric": "mean",
      "higher_is_better": false,
      "unit": "fraction",
      "aggregation": "median"
    },
    {
      "id": "exact_match",
      "metric": "mean",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "median"
    },
    {
      "id": "confidence_low",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "confidence_medium",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "confidence_high",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "seed_template_accuracy",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "ambiguous_accuracy",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "long_thread_accuracy",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "non_english_accuracy",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    },
    {
      "id": "no_detail_accuracy",
      "metric": "precision",
      "higher_is_better": true,
      "unit": "fraction",
      "aggregation": "summed_counts"
    }
  ],
  "repeats": 3,
  "seed": 20261027,
  "verdict_rule": {
    "bootstrap": "paired-items",
    "resamples": 10000,
    "confidence": 0.95,
    "prng": "mulberry32",
    "quantile": "type-7",
    "range": "absolute-difference-exceeds-wider-run-range",
    "label_tie": "json-lexicographic",
    "zero_division": 0,
    "degenerate_kappa": 1,
    "rounding": "half-away-from-zero",
    "decimals": 4
  },
  "record_shape": [
    "gold",
    "prediction"
  ],
  "prompt": "You triage customer support tickets. Read the ticket, then call triage_ticket exactly once. category: what the customer needs. billing: charges, invoices, refunds, payment, subscription cost. access: sign-in, locks, permissions, identity, lost access. bug: a feature that exists fails or behaves wrongly. feature: a request for new or changed capability. unknown: too little information to classify; use this instead of guessing. priority: P1 only when work is blocked with no workaround; P2 when the customer's daily work is affected but the workspace works; P3 otherwise, including wishes and vague messages. team: finance for billing, identity for access, engineering for bug, product for feature, support for unknown. confidence: low when the message is vague, short, non-English or contradictory; medium when one field is uncertain; high otherwise. reason: one sentence citing the ticket.",
  "tuning_repeats": 1,
  "tool_choice": {
    "type": "function",
    "function": {
      "name": "triage_ticket"
    }
  },
  "max_tokens": 2048,
  "reasoning_effort": "low",
  "fixture_sha256": "01acfa159c41cdb5c1f3cf091c3b2103663bbb272273c9ba37cf754ee4ecd938",
  "recipe_sha256": "290d03c422efaa056f1ab534066351cb236e7e174f5ad9f2132e7f25f17335f3",
  "runner_sha256": "062009bf04fbbaed1f882d94325ccf60488dedafb1f9447e4f5a7146e77563a2",
  "checker_sha256": "359fb78b9a0a4749972a7e0bdf91dd0753239c3659984ae1b36053f51666aeb6",
  "fixture_source": {
    "source": "130 synthetic tickets written for this evaluation",
    "license": "CC BY 4.0",
    "generators": [
      "Pareto coding agent; deterministic Python templates"
    ]
  },
  "failure_label": "invalid"
}