{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/meeting-transcript-to-action-items/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/meeting-transcript-to-action-items/fixture.jsonl",
      "sha256": "e46ab9e6b27863846e6e29b3b59f0a20b2962649553ebe8a73e0ba51144de5d3",
      "items": 10,
      "dev_items": 2,
      "split": "test",
      "source": "Original fictional meetings from our deterministic generator",
      "license": "CC0",
      "generators": [
        "Pareto (generator code author; deterministic rendering)"
      ]
    },
    "prompt_sha256": "be4faeb50fc5b72e12ecad602d2ff5f4fcf69916f25dfd919db3c28e17975a10",
    "recipe_sha256": "9b46132dca9aadf909cebe8791a90fee5980c1a693f1713bf2311709946ff74a",
    "eval_json_sha256": "cd9b325e9328d0d973a1bf276fbf22f4c57e2d72fbf811632b74ee3b6717c34f",
    "freeze_sha": "9b9151a86d24b2c1359edc53c66166fd8a1b5d45",
    "repeats": 3,
    "seed": 42,
    "unit": "meeting",
    "aggregation": "summed_counts",
    "headline": {
      "metric": "f1",
      "higher_is_better": true,
      "unit": "fraction"
    },
    "secondary": [
      {
        "id": "action_precision",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "action_recall",
        "metric": "recall",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "decision_precision",
        "metric": "precision",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "decision_recall",
        "metric": "recall",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "owner_not_speaker_recall",
        "metric": "recall",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "invented_dates",
        "metric": "count",
        "higher_is_better": false,
        "unit": "dates",
        "aggregation": "median"
      },
      {
        "id": "invented_date_rate",
        "metric": "precision",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "summed_counts"
      },
      {
        "id": "action_items_found",
        "metric": "count",
        "higher_is_better": true,
        "unit": "actions",
        "aggregation": "median"
      },
      {
        "id": "gold_action_items",
        "metric": "count",
        "higher_is_better": true,
        "unit": "actions",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 8192,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 59775,
              "output": 2311,
              "cached": 2831,
              "reasoning": 564
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "acd93aaf9f859eaa1e545c443819e37ef7c02d21c1be2d1aef95ef5e1f13bddf"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 59775,
              "output": 2290,
              "cached": 58445,
              "reasoning": 543
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "46d24f985fb132223460a8b2cb062fab15b7c543e1489bf0b693e3da4710ed6e"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 59775,
              "output": 2296,
              "cached": 58445,
              "reasoning": 531
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "1b135ed1173f802699ed6c0442cef9b0c5634ed67303b5f9ba48283ed187541b"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 8192,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 63277,
              "output": 19466,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "1cd28416144e3a8046b1753e76b634da210834d833e636c873b3449a4d6f9050"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 63277,
              "output": 17803,
              "cached": 54470,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "4dd6656175f94960f932e07fd83ee9eeeaecfe6be6913eb72ab6a4509a8c271f"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 63277,
              "output": 20789,
              "cached": 54470,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "39bbf7c48100bc3b447254720166889bbe058e300a16a4160c9eaba2990ca8db"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/meeting-transcript-to-action-items/runs/2026-10-28T00-10-41-782820Z/raw.jsonl.gz",
      "sha256": "9af0c6b7598b963b6ba81d1235c7e32e98c277d054d8a979180f8e77f3da4369"
    }
  }
}