{
  "date": "2026-10-28",
  "result": "pass",
  "eval": {
    "schema": 2,
    "runner": "scripts/evals/app-string-localization/run_eval.py",
    "pareto_release": "pareto-26.10",
    "fixture": {
      "path": "data/use-case-evals/app-string-localization/fixture.jsonl",
      "sha256": "5f4971588347fa747a2b752696afa412c46dbc1f7a8dd34feba7660cf4ac59f9",
      "items": 1000,
      "dev_items": 200,
      "split": "test",
      "source": "Original synthetic application strings; generator-intended structural labels",
      "license": "CC0-1.0",
      "generators": [
        "Pareto (authoring assistant), deterministic Python renderer"
      ]
    },
    "prompt_sha256": "9c58e058d8be8a888ccd5e156088289ca95071e531d0f675e38ccaaa3f58c6c9",
    "recipe_sha256": "c7050d65dbfed559333932be93898e7d4115e7fd5da61b61756b659f41a5af1b",
    "eval_json_sha256": "58ee405c73c80738a75a4511d4314c2b240eb31fa2afae689ad1bbd5be4173dd",
    "freeze_sha": "b3c14c74568a7af10dcbfb3dd44a370a2a655ba3",
    "repeats": 3,
    "seed": 42,
    "unit": "string x locale",
    "aggregation": "median",
    "headline": {
      "metric": "mean",
      "higher_is_better": false,
      "unit": "fraction"
    },
    "secondary": [
      {
        "id": "icu_valid",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "plural_complete",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "glossary_adherence",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "key_set_equal",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "markup_preserved",
        "metric": "mean",
        "higher_is_better": true,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "after_retry",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "locale_de",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "locale_ja",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "locale_pl",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "locale_ar",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      },
      {
        "id": "locale_pt_BR",
        "metric": "mean",
        "higher_is_better": false,
        "unit": "fraction",
        "aggregation": "median"
      }
    ],
    "arms": {
      "baseline": {
        "model": "openai/gpt-6.1-sol",
        "endpoint": "https://openrouter.ai/api/v1",
        "max_tokens": 512,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 210940,
              "output": 46669,
              "cached": 0,
              "reasoning": 8486
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "1467053e15f7ec8484ac48b5a3600d1de84ba960572130e259d1af92e744458b"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 210940,
              "output": 45352,
              "cached": 0,
              "reasoning": 7222
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "913bcf481d9cec4cb46f6714a7a76c3bc10030217d21bfa85e9342f440d276d9"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 210940,
              "output": 45741,
              "cached": 0,
              "reasoning": 7600
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "604b46237540b4fb27344959d603843f1b5c9c16c19fb4137d738f2ef6125831"
          }
        ],
        "label": "GPT-6.1 Sol",
        "provider_served": "Azure",
        "reasoning_effort": "medium"
      },
      "pareto": {
        "model": "pareto",
        "endpoint": "https://api.unbiased.ai/v1",
        "max_tokens": 512,
        "runs": [
          {
            "repeat": 1,
            "tokens": {
              "input": 323565,
              "output": 34530,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "f403ec10fa3239027c75d74f3e0ad9972f80f454d7e8eeaf8b5e152c0f203ee7"
          },
          {
            "repeat": 2,
            "tokens": {
              "input": 323966,
              "output": 34602,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "5b0533332624f8b116f2f0b00dea03c12e712f05753201755c1aca2cab5a4f24"
          },
          {
            "repeat": 3,
            "tokens": {
              "input": 323565,
              "output": 34588,
              "cached": 0,
              "reasoning": 0
            },
            "errors": 0,
            "invalid_tool_calls": 0,
            "retries": 0,
            "truncations": 0,
            "response_ids_sha": "ad2ecb4e48a8703ca2831e6b13fd1cb4e8804e60d94379010578e0a57658ad5e"
          }
        ],
        "response_model": "pareto-26.10"
      }
    },
    "variants": {},
    "raw": {
      "path": "data/use-case-evals/app-string-localization/runs/2026-10-28T01-33-53-603540Z/raw.jsonl.gz",
      "sha256": "761b697b7eb11ac089a04fae369b444fa3faa95b28162a1b6de25a43d28457e0"
    }
  }
}