#!/usr/bin/env python3
"""The Unbiased Verbosity Index battery. https://unbiased.ai/blog/verbosity-index/
Identical prompts, no length guidance, 2 runs per model per task, default settings.
Set ANTHROPIC_API_KEY (and PARETO_API_KEY / PARETO_BASE_URL to include Pareto).
If your run disagrees with a published edition, publish it."""
import json, re, subprocess, time, urllib.request

import os
ANT_KEY = os.environ["ANTHROPIC_API_KEY"]
PAR_KEY = os.environ.get("PARETO_API_KEY", "")
PAR_BASE = os.environ.get("PARETO_BASE_URL", "")  # provided with Pareto credits at onboarding

MODELS = {  # name: (api, model_id, $in/M, $out/M)
 "Claude Fable 5":   ("ant", "claude-fable-5", 10, 50),
 "Claude Opus 5":    ("ant", "claude-opus-5", 5, 25),
 "Claude Opus 4.8":  ("ant", "claude-opus-4-8", 5, 25),
 "Claude Sonnet 5":  ("ant", "claude-sonnet-5", 2, 10),
 "Claude Haiku 4.5": ("ant", "claude-haiku-4-5-20251001", 1, 5),
 "Pareto":           ("par", "pareto", 0, 0),
}
TASKS = {
 "codegen": "Build a single-file HTML page: a sortable data table of 12 SaaS invoices (vendor, date, amount, status) with a summary bar showing total spend and count of overdue items. Clean light theme, all CSS and JS inline, no external libraries. Return only the complete HTML file in one code block.",
 "extraction": "Extract structured data from this invoice text as JSON with fields vendor, invoice_number, date, line_items (array of {description, qty, unit_price, total}), subtotal, tax, total. Text: ACME CLOUD SERVICES - Invoice #AC-2026-0847, July 14 2026. GPU compute A100 80GB, 220 hrs @ $2.40/hr = $528.00. Object storage, 14 TB-months @ $21.00 = $294.00. Egress bandwidth, 3.2 TB @ $85.00/TB = $272.00. Support plan (business) = $400.00. Subtotal $1,494.00, Tax (8.875%) $132.59, TOTAL DUE $1,626.59. Return only the JSON.",
 "summarize": "Summarize this in three sentences: The city council voted 7-2 on Tuesday to approve the riverfront redevelopment plan after four hours of public comment. The $340 million project will replace the shuttered textile mill with 1,200 housing units, a public park, and retail space, with construction beginning in March. Opponents raised concerns about displacement of the artists' collective currently leasing the mill, traffic on Route 9, and the 30-year tax increment financing arrangement. The council added conditions requiring 15% affordable units, a relocation fund for current tenants, and a traffic study before phase two. The developer, Meridian Partners, called the conditions workable and said pre-leasing would begin next fall.",
 "analysis": "A SaaS company has 1,400 customers paying $89/month. Monthly churn is 2.1%. CAC is $640, gross margin 78%. Calculate LTV using margin-adjusted revenue, the LTV:CAC ratio, and months to recover CAC. Then state in two sentences whether they should increase paid acquisition spend. Show your arithmetic.",
 "email": "Write a professional email to a vendor asking to renegotiate our contract renewal: we have been a customer for 3 years, usage grew 4x, and a competitor quoted 30% less. We want to stay but need them to close most of the gap. Keep it firm and warm.",
 "short_answer": "What temperature does water freeze at, in Fahrenheit and Celsius?",
}

def call_ant(model, prompt):
    body = json.dumps({"model":model,"max_tokens":16000,"messages":[{"role":"user","content":prompt}]}).encode()
    req = urllib.request.Request("https://api.anthropic.com/v1/messages", data=body,
        headers={"x-api-key":ANT_KEY,"anthropic-version":"2023-06-01","content-type":"application/json"})
    with urllib.request.urlopen(req, timeout=600) as r: d=json.loads(r.read())
    u=d["usage"]; return u["input_tokens"], u["output_tokens"], None

def call_par(prompt):
    body = json.dumps({"model":"pareto","messages":[{"role":"user","content":prompt}]}).encode()
    req = urllib.request.Request(PAR_BASE + "/chat/completions", data=body,
        headers={"Authorization":f"Bearer {PAR_KEY}","Content-Type":"application/json"})
    with urllib.request.urlopen(req, timeout=600) as r: d=json.loads(r.read())
    u=d["usage"]; meta=(d.get("_meta") or {}).get("cascade") or {}
    return u["prompt_tokens"], u["completion_tokens"], meta.get("cost_usd")

results = {}
for mname, (api, mid, pi, po) in MODELS.items():
    for tname, prompt in TASKS.items():
        for run in (1, 2):
            key = f"{mname}|{tname}|{run}"
            try:
                if api == "ant":
                    ti, to, cost = call_ant(mid, prompt)
                    cost = ti/1e6*pi + to/1e6*po
                else:
                    ti, to, cost = call_par(prompt)
                results[key] = {"in": ti, "out": to, "cost": round(cost or 0, 6)}
                print(key, results[key], flush=True)
            except Exception as e:
                print(key, "ERR", str(e)[:100], flush=True)
            time.sleep(0.8)
json.dump(results, open("verbosity-results.json","w"), indent=1)
print("BATTERY_DONE", len(results), "runs")