{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/ai-unit-test-generation/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/ai-unit-test-generation/fixture.jsonl",
      "sha256": "ed78fc258cb79077a4ec3cd017b6506061dedf7f0d0bc018360cd565a723899d",
      "items": 60,
      "dev_items": 20,
      "split": "test",
      "source": "Original MIT pure-Python modules written for this page",
      "license": "MIT",
      "generators": [
        "Pareto coding agent"
      ]
    },
    "prompt_sha256": "6a04a42700df16de6f7f5336adbc1b1d74d27b1eecff7e79ac75f6200216868f",
    "recipe_sha256": "3e832df4367347f054134428cfaa9ebad9709dce8cf513c0dd857ea6343de265",
    "eval_json_sha256": "d5f4ce8f70edfb3af8cdd7de44c4fce3a2fd7d56b0fde4b8ba7e20900d8128be",
    "freeze_sha": "063a17639a633ba548c624cc7693af9416b740e9",
    "repeats": 3,
    "seed": 17,
    "unit": "bug",
    "aggregation": "median",
    "headline": {
      "metric": "count",
      "higher_is_better": true,
      "unit": "seeded bugs caught"
    },
    "secondary": [
      {
        "id": "clean_pass",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "line_coverage",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "branch_coverage",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "mutation_score",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 32084,
              "output": 75721,
              "cached": 0,
              "reasoning": 5547
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "785a11a23a6f7249e5480e14a44ecc9293c07faf9f8fcc22272518e060bf803c"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 32084,
              "output": 75886,
              "cached": 0,
              "reasoning": 5525
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "3038e299ed3c57ef90dad5129aae4e686ef55b861cb95ec766d8cde8555063d6"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 32084,
              "output": 75557,
              "cached": 0,
              "reasoning": 5573
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "0a8c2b4c200979cffb929d1b3336ed0f4114566bd54fd4ee47fa79c059211340"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 4096,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 63268,
              "output": 283390,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "5cffa9fe7fe8eeb2f8594379d7c2fcbafae2d6a8ff1212b5a742fc6d747a99ca"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 61138,
              "output": 278377,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 1,
            "retries": 0,
            "truncations": 1,
            "response_ids_sha": "0578c78e5a9e7231e38298ded9bc300b594a382dc7aa19b874482cab258ac803"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 61205,
              "output": 280107,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "c92866fc7d372f9f17200fd7c9cd07e77d548be500d16d180e14a41e0ffd0ded"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/ai-unit-test-generation/runs/2026-10-28T01-04-14-745473Z/raw.jsonl.gz",
      "sha256": "54b9a06bcd361e265d33c9dac734a5a3a61352dff1d150afcce5dfd9f8f62e84"
    }
  }
}