{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/text-to-sql/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/text-to-sql/fixture.jsonl",
      "sha256": "72ee7658af15cba8baf700159088d1d711ef25aa3cc70b4e679be8f4a672f454",
      "items": 50,
      "dev_items": 15,
      "split": "test",
      "source": "Generator-intended questions on Chinook schema with new synthetic seed data; labels set by the generator, spot-checked by us (automated checks, human review pending)",
      "license": "CC0",
      "generators": [
        "Pareto (authoring agent; SQL intent fixed before question text)",
        "Python deterministic seed generator"
      ]
    },
    "prompt_sha256": "ad6a4a73729b44e1876778dc8801fb149d194fc933ee7442b489630cc3175eeb",
    "recipe_sha256": "72d594474b5b0f2f26ef67639211e4bbb5b9a65a4c4f9fdfed3c051538142899",
    "eval_json_sha256": "c37a506fb6b76a3ace106b8bb1e5ccf8e9a79e419fc2e3f3fd5390b5d791431c",
    "freeze_sha": "daa4cbcf5d041d843f2cd1bf84a28cdef386007b",
    "repeats": 3,
    "seed": 20261027,
    "unit": "question",
    "aggregation": "median",
    "headline": {
      "metric": "mean",
      "higher_is_better": true,
      "unit": "first-pass execution accuracy"
    },
    "secondary": [
      {
        "id": "invalid_sql_before_retry",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "invalid_sql_after_retry",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "accuracy_after_retry",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "easy_correct",
        "metric": "count",
        "higher_is_better": true,
        "unit": "correct of twenty easy questions",
        "aggregation": "median"
      },
      {
        "id": "medium_correct",
        "metric": "count",
        "higher_is_better": true,
        "unit": "correct of twenty medium questions",
        "aggregation": "median"
      },
      {
        "id": "hard_correct",
        "metric": "count",
        "higher_is_better": true,
        "unit": "correct of ten hard questions",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 2048,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 67713,
              "output": 4435,
              "cached": 0,
              "reasoning": 180
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "54231826b0a9b1f57f23177a4b89e1a3729825fc4c7a543160d4968757a7250b"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 67713,
              "output": 4396,
              "cached": 61113,
              "reasoning": 149
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "b9c0b5e1f66e65cf71796aa74a1960f0520bd16e6a8132ae5051b0ef72072b5b"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 67713,
              "output": 4342,
              "cached": 61113,
              "reasoning": 110
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "cca0ad1bbde58e22f291f79a331f4de65056bcc8309837b17e753cfeeda9670d"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 2048,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 72062,
              "output": 12860,
              "cached": 29150,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "adf0522ef77fbd8f3afddeff3c851a46f14f9c03a76849366d2a44e2efb9ef20"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 72062,
              "output": 14796,
              "cached": 29150,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "7d64647660cb953a927f11b298f1cdb747a18bc54d4ae10acf578929c3f292c6"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 72062,
              "output": 11775,
              "cached": 29150,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "b24eafc31e2e94adbb6374fb32e8d45747bde95316882f2447b679fa4376642a"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/text-to-sql/runs/2026-10-28T00-10-20-244393Z/raw.jsonl.gz",
      "sha256": "bfb00a618a8b6e9407a4d5cc20f3e84cee02348cf2e4121f2fb90f4ee9fcdf4a"
    }
  }
}