#!/usr/bin/env python3 """Build the static, auditable dataset consumed by the leaderboard site.""" from __future__ import annotations import json import shutil import sys from collections import Counter, defaultdict from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Any, Iterable ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) from pokemon_bench.models import BestOfThreeResult, TournamentSummary try: from scripts.bradley_terry import fit_results except ModuleNotFoundError: # Direct `python scripts/build_site_data.py` execution. from bradley_terry import fit_results OUTPUT = ROOT / "site" / "src" / "data" / "benchmark-data.json" EVIDENCE_DIR = ROOT / "site" / "public" / "evidence" RANKED_SUMMARIES = ( ROOT / "runs" / "openai-six-mirrored-season-1-scenario-1" / "summary.json", ROOT / "runs" / "cross-provider-anchored-season-1-recovery-2" / "summary.json", ROOT / "runs" / "gpt-6.1-sol-anchored-season-1" / "summary.json", ROOT / "runs" / "gpt-6.1-sol-anchored-season-2" / "summary.json", ) EXPERIMENTAL_RUN = ROOT / "runs" / "system-one-anchored-season-1" def system_one_directories() -> list[Path]: return [EXPERIMENTAL_RUN, *sorted(path for path in (ROOT / "runs").glob("system-one-precision-*-round-*") if (path / "manifest.json").is_file())] DISPLAY_NAMES = { "gpt-6-astra": "GPT-6 Astra", "gpt-6.1-sol": "GPT-6.1 Sol", "gpt-6-sol": "GPT-6 Sol", "gpt-6-luna": "GPT-6 Luna", "gpt-5.6-sol": "GPT-5.6 Sol", "gpt-5.6-terra": "GPT-5.6 Terra", "gpt-5.6-luna": "GPT-5.6 Luna", "claude-fable-5-1": "Claude Fable 5.1", "claude-opus-5-5": "Claude Opus 5.5", "gemini-3.8-flash": "Gemini 3.8 Flash", "muse-spark-1.3": "Muse Spark 1.3", "kimi-k3": "Kimi K3", "glm-5.3": "GLM 5.3", "grok-4.7": "Grok 4.7", "jev-1.13.0": "Jev 1.13", "clef": "Clef", } PROVIDERS = { "gpt-6-astra": "OpenAI", "gpt-6.1-sol": "OpenAI", "gpt-6-sol": "OpenAI", "gpt-6-luna": "OpenAI", "gpt-5.6-sol": "OpenAI", "gpt-5.6-terra": "OpenAI", "gpt-5.6-luna": "OpenAI", "claude-fable-5-1": "Anthropic", "claude-opus-5-5": "Anthropic", "gemini-3.8-flash": "Google", "muse-spark-1.3": "Meta", "kimi-k3": "Moonshot AI", "glm-5.3": "Z.ai", "grok-4.7": "xAI", "jev-1.13.0": "TypeSafe AI", "clef": "Cloudflare", } @dataclass(frozen=True) class Price: input_per_million: float cached_input_per_million: float output_per_million: float source: str source_label: str note: str = "" cached_is_subset: bool = True reasoning_is_separate: bool = False PRICES = { "gpt-6-astra": Price(10, 1, 50, "https://developers.openai.com/api/docs/models/gpt-6-astra", "OpenAI pricing"), "gpt-6.1-sol": Price(2, 0.1, 10, "https://developers.openai.com/api/docs/models/gpt-6.1-sol", "OpenAI pricing"), "gpt-6-sol": Price(2, 0.2, 10, "https://developers.openai.com/api/docs/models/gpt-6-sol", "OpenAI pricing"), "gpt-6-luna": Price(0.1, 0.01, 0.5, "https://developers.openai.com/api/docs/models/gpt-6-luna", "OpenAI pricing"), "gpt-5.6-sol": Price(4, 0.4, 20, "https://developers.openai.com/api/docs/models/gpt-5.6-sol", "OpenAI pricing"), "gpt-5.6-terra": Price(2, 0.2, 12, "https://developers.openai.com/api/docs/models/gpt-5.6-terra", "OpenAI pricing"), "gpt-5.6-luna": Price(0.2, 0.02, 1.2, "https://developers.openai.com/api/docs/models/gpt-5.6-luna", "OpenAI pricing"), "claude-fable-5-1": Price(10, 0.25, 50, "https://www.anthropic.com/claude-fable-and-mythos-5-1", "Anthropic announcement", cached_is_subset=False), "claude-opus-5-5": Price(4, 0.2, 20, "https://www.anthropic.com/claude-opus-5-5", "Anthropic announcement", cached_is_subset=False), "gemini-3.8-flash": Price(0.75, 0.075, 3.75, "https://ai.google.dev/gemini-api/docs/pricing", "Google pricing", "Introductory pricing through 2026-12-31.", reasoning_is_separate=True), "muse-spark-1.3": Price(1.25, 0.15, 4.25, "https://dev.meta.ai/docs/pricing-rate-limits", "Meta pricing"), "kimi-k3": Price(3, 0.3, 15, "https://platform.moonshot.ai/docs/pricing/chat-k3", "Moonshot pricing"), "glm-5.3": Price(1.4, 0.14, 4.4, "https://docs.z.ai/guides/overview/pricing", "Z.ai pricing"), "grok-4.7": Price(2, 0.5, 6, "https://docs.x.ai/docs/models/grok-4-7", "xAI pricing", "Short-context standard rate."), "jev-1.13.0": Price(0.042, 0.042, 0, "https://docs.typesafe.ai/models", "TypeSafe AI pricing", "Input tokens only."), } def load_ranked_results() -> tuple[list[BestOfThreeResult], list[dict[str, Any]]]: results: list[BestOfThreeResult] = [] runs: list[dict[str, Any]] = [] for path in RANKED_SUMMARIES: summary = TournamentSummary.model_validate_json(path.read_text(encoding="utf-8")) manifest = json.loads((path.parent / "manifest.json").read_text(encoding="utf-8")) results.extend(summary.results) runs.append( { "id": summary.tournament_id, "status": "complete", "completedSeries": summary.completed_legs, "plannedSeries": summary.planned_legs, "protocol": "joint-action-json-schema-v1", "createdAt": manifest["created_at"], "sourceHash": manifest["engine"]["benchmark_source_sha256"], "failedAttempts": summary.failed_attempts, "evidencePath": f"/evidence/{summary.tournament_id}.json", } ) return results, runs def load_experimental_results(directory: Path = EXPERIMENTAL_RUN) -> tuple[list[BestOfThreeResult], dict[str, Any]]: paths = sorted(directory.glob("pair-*/leg-*.result.json")) results = [ BestOfThreeResult.model_validate_json(path.read_text(encoding="utf-8")) for path in paths ] manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) return results, { "id": manifest["tournament_id"], "status": "complete" if len(results) == sum(len(pair["legs"]) for pair in manifest["pairings"]) else "incomplete", "completedSeries": len(results), "plannedSeries": sum(len(pair["legs"]) for pair in manifest["pairings"]), "protocol": "system-one-choice-tournament-v1", "createdAt": manifest["created_at"], "sourceHash": manifest["engine"]["benchmark_source_sha256"], "failedAttempts": sum(1 for line in (directory / "failures.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()) if (directory / "failures.jsonl").is_file() else 0, "evidencePath": f"/evidence/{manifest['tournament_id']}.json", } def game_records(results: Iterable[BestOfThreeResult]) -> dict[str, Counter[str]]: records: dict[str, Counter[str]] = defaultdict(Counter) for result in results: for model in result.models: records[model]["series"] += 1 records[model]["games"] += len(result.games) records[result.winner_model]["series_wins"] += 1 for game in result.games: records[game.winner_model]["game_wins"] += 1 return records def iter_decisions(path: Path, *, strict: bool = False, audit: Counter[str] | None = None) -> Iterable[dict[str, Any]]: try: with path.open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, 1): try: record = json.loads(line) except json.JSONDecodeError as exc: if strict: raise ValueError(f"Malformed JSON in {path}:{line_number}") from exc if audit is not None: audit["skippedRecords"] += 1 continue if record.get("record_type") != "action_decision": continue decisions = record.get("decisions") if isinstance(decisions, dict): yield from decisions.values() except OSError as exc: if strict: raise RuntimeError(f"Cannot read required trajectory: {path}") from exc if audit is not None: audit["skippedFiles"] += 1 def aggregate_usage(paths: Iterable[Path], *, strict: bool = False) -> tuple[dict[str, Counter[str]], Counter[str]]: usage: dict[str, Counter[str]] = defaultdict(Counter) audit: Counter[str] = Counter() for path in sorted(paths): audit["filesRead"] += 1 for decision in iter_decisions(path, strict=strict, audit=audit): model = decision.get("model") measured = decision.get("usage") if not isinstance(model, str) or not isinstance(measured, dict): continue usage[model]["calls"] += 1 for field in ("input_tokens", "cached_input_tokens", "output_tokens", "reasoning_tokens", "total_tokens"): value = measured.get(field) if isinstance(value, int): usage[model][field] += value return usage, audit def ranked_trajectory_paths() -> list[Path]: expected_paths: list[Path] = [] discovered_paths: list[Path] = [] for summary_path in RANKED_SUMMARIES: summary = TournamentSummary.model_validate_json(summary_path.read_text(encoding="utf-8")) expected_ids = [result.match_id for result in summary.results] result_paths = sorted(summary_path.parent.glob("pair-*/leg-[12].result.json")) result_by_id: dict[str, Path] = {} for result_path in result_paths: result = BestOfThreeResult.model_validate_json(result_path.read_text(encoding="utf-8")) if result.match_id in result_by_id: raise RuntimeError(f"Duplicate result identity {result.match_id} in {summary_path.parent}") result_by_id[result.match_id] = result_path if set(expected_ids) != set(result_by_id): missing = sorted(set(expected_ids) - set(result_by_id)) unexpected = sorted(set(result_by_id) - set(expected_ids)) raise RuntimeError(f"Result identity mismatch in {summary_path.parent}: missing={missing}, unexpected={unexpected}") expected_paths.extend( result_by_id[match_id].with_suffix("").with_suffix(".jsonl") for match_id in expected_ids ) discovered_paths.extend(summary_path.parent.glob("pair-*/leg-[12].jsonl")) if set(expected_paths) != set(discovered_paths): missing = sorted(set(expected_paths) - set(discovered_paths)) unexpected = sorted(set(discovered_paths) - set(expected_paths)) raise RuntimeError(f"Ranked trajectory mismatch: missing={missing}, unexpected={unexpected}") return sorted(expected_paths) def experimental_trajectory_paths(results: Iterable[BestOfThreeResult], directory: Path = EXPERIMENTAL_RUN) -> list[Path]: expected_ids = [result.match_id for result in results] result_paths = sorted(directory.glob("pair-*/leg-[12].result.json")) result_by_id: dict[str, Path] = {} for result_path in result_paths: result = BestOfThreeResult.model_validate_json(result_path.read_text(encoding="utf-8")) if result.match_id in result_by_id: raise RuntimeError(f"Duplicate experimental result identity {result.match_id}") result_by_id[result.match_id] = result_path if set(expected_ids) != set(result_by_id): missing = sorted(set(expected_ids) - set(result_by_id)) unexpected = sorted(set(result_by_id) - set(expected_ids)) raise RuntimeError(f"Experimental result identity mismatch: missing={missing}, unexpected={unexpected}") expected_paths = [ result_by_id[match_id].with_suffix("").with_suffix(".jsonl") for match_id in expected_ids ] discovered_paths = list(directory.glob("pair-*/leg-[12].jsonl")) if set(expected_paths) != set(discovered_paths): missing = sorted(set(expected_paths) - set(discovered_paths)) unexpected = sorted(set(discovered_paths) - set(expected_paths)) raise RuntimeError(f"Experimental trajectory mismatch: missing={missing}, unexpected={unexpected}") return sorted(expected_paths) def merge_usage(*groups: dict[str, Counter[str]]) -> dict[str, Counter[str]]: merged: dict[str, Counter[str]] = defaultdict(Counter) for group in groups: for model, measured in group.items(): merged[model].update(measured) return merged def aggregate_resolved_models(paths: Iterable[Path]) -> dict[str, set[str]]: resolved: dict[str, set[str]] = defaultdict(set) for path in sorted(paths): for decision in iter_decisions(path): model = decision.get("model") version = decision.get("resolved_model") if isinstance(model, str) and isinstance(version, str) and version: resolved[model].add(version) return resolved def estimate_cost(model: str, usage: Counter[str]) -> float | None: price = PRICES.get(model) if price is None: return None input_tokens = usage["input_tokens"] cached_tokens = usage["cached_input_tokens"] uncached_tokens = max(input_tokens - cached_tokens, 0) if price.cached_is_subset else input_tokens billable_output = usage["output_tokens"] if price.reasoning_is_separate: billable_output += usage["reasoning_tokens"] return ( uncached_tokens * price.input_per_million + cached_tokens * price.cached_input_per_million + billable_output * price.output_per_million ) / 1_000_000 def usage_totals(group: dict[str, Counter[str]]) -> tuple[Counter[str], float]: total = Counter() cost = 0.0 for model, measured in group.items(): total.update(measured) cost += estimate_cost(model, measured) or 0 return total, cost def write_evidence_files() -> None: """Publish compact manifests and outcomes, never the large trajectories.""" EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) evidence_runs: list[dict[str, Any]] = [] for summary_path in RANKED_SUMMARIES: manifest = json.loads((summary_path.parent / "manifest.json").read_text(encoding="utf-8")) summary = json.loads(summary_path.read_text(encoding="utf-8")) evidence_runs.append({"manifest": manifest, "summary": summary}) for directory in system_one_directories(): manifest = json.loads((directory / "manifest.json").read_text(encoding="utf-8")) results = [json.loads(path.read_text(encoding="utf-8")) for path in sorted(directory.glob("pair-*/leg-[12].result.json"))] evidence_runs.append({"manifest": manifest, "completed_results": results}) precision_plans = [ {"name": path.parent.name, "files": {item.name: json.loads(item.read_text(encoding="utf-8")) for item in sorted(path.parent.glob("*.json"))}} for path in sorted((ROOT / "runs").glob("system-one-precision-*-planning/baseline.json")) ] index = { "schema_version": "pokemon-bench-public-evidence-v1", "note": "Compact manifests and completed outcomes; raw JSONL trajectories are excluded.", "runs": evidence_runs, "precisionSampling": precision_plans, } (EVIDENCE_DIR / "index.json").write_text( json.dumps(index, indent=2) + "\n", encoding="utf-8" ) for run in evidence_runs: tournament_id = run["manifest"]["tournament_id"] (EVIDENCE_DIR / f"{tournament_id}.json").write_text( json.dumps(run, indent=2) + "\n", encoding="utf-8" ) source_dir = EVIDENCE_DIR / "source" source_dir.mkdir(exist_ok=True) public_sources = { OUTPUT: "benchmark-data.json", ROOT / "scripts" / "build_site_data.py": "build_site_data.py.txt", ROOT / "scripts" / "bradley_terry.py": "bradley_terry.py.txt", ROOT / "src" / "pokemon_bench" / "prompts" / "battle-policy-v3.md": "battle-policy-v3.md.txt", ROOT / "src" / "pokemon_bench" / "schema.py": "schema.py.txt", ROOT / "examples" / "system_one_precision.py": "system_one_precision.py.txt", } for source, filename in public_sources.items(): shutil.copyfile(source, source_dir / filename) def head_to_head(results: Iterable[BestOfThreeResult]) -> list[dict[str, Any]]: """Directed row-versus-column records, using completed canonical series only.""" pairs: dict[tuple[str, str], Counter[str]] = defaultdict(Counter) for result in results: for model in result.models: opponent = next(other for other in result.models if other != model) record = pairs[model, opponent] record["series"] += 1 record["seriesWins"] += result.winner_model == model record["games"] += len(result.games) record["gameWins"] += sum(game.winner_model == model for game in result.games) return [{"model": model, "opponent": opponent, **record} for (model, opponent), record in sorted(pairs.items())] def main() -> None: ranked_results, runs = load_ranked_results() experimental_results, experimental_run = load_experimental_results() system_one_results = experimental_results runs.append(experimental_run) system_one_paths = experimental_trajectory_paths(system_one_results) for directory in system_one_directories()[1:]: results, run = load_experimental_results(directory) system_one_results = [*system_one_results, *results] system_one_paths.extend(experimental_trajectory_paths(results, directory)) runs.append(run) experimental_results = system_one_results # Completed System One series belong to the same connected match graph. # Preserve interface metadata and fit all systems with the same weak prior. system_one_models = {model for result in experimental_results for model in result.models} - {model for result in ranked_results for model in result.models} protocols = {model: {'system-one-choice-tournament-v1'} for model in system_one_models} protocols.update({model: {'joint-action-json-schema-v1'} for result in ranked_results for model in result.models}) ranked_results = [*ranked_results, *experimental_results] experimental_results = [] ratings = fit_results(ranked_results, protocols, prior_std=2.0)["ratings"] rating_by_model = {row["model"]: row for row in ratings} ranked_records = game_records(ranked_results) experimental_records = game_records(experimental_results) ranked_paths = [*ranked_trajectory_paths(), *system_one_paths] experimental_paths: list[Path] = [] ranked_usage, ranked_audit = aggregate_usage(ranked_paths, strict=True) experimental_usage, experimental_audit = aggregate_usage(experimental_paths, strict=True) evaluated_usage = merge_usage(ranked_usage, experimental_usage) paid_usage, paid_audit = aggregate_usage((ROOT / "runs").glob("**/*.jsonl")) resolved_models = aggregate_resolved_models([*ranked_paths, *experimental_paths]) ranked_models = {model for result in ranked_results for model in result.models} experimental_models = {model for result in experimental_results for model in result.models} - ranked_models models = sorted(ranked_models | experimental_models) reasoning_levels: dict[str, set[str]] = defaultdict(set) planned_series: Counter[str] = Counter() for path in [*(path.parent / "manifest.json" for path in RANKED_SUMMARIES), *(directory / "manifest.json" for directory in system_one_directories())]: manifest = json.loads(path.read_text(encoding="utf-8")) for pair in manifest["pairings"]: for model in pair["models"]: planned_series[model] += len(pair["legs"]) for model, config in manifest["model_configs"].items(): reasoning_levels[model].add(config.get("reasoning_effort", "unspecified")) leaderboard = [] for model in models: rating = rating_by_model.get(model) is_ranked = model in ranked_models record = ranked_records[model] if is_ranked else experimental_records[model] token_usage = ranked_usage[model] if is_ranked else experimental_usage[model] paid_token_usage = paid_usage[model] evaluated_cost = estimate_cost(model, token_usage) paid_cost = estimate_cost(model, paid_token_usage) leaderboard.append( { "id": model, "name": DISPLAY_NAMES.get(model, model), "provider": PROVIDERS.get(model, "Unknown"), "reasoningLevels": sorted(reasoning_levels[model]), "plannedSeries": planned_series[model], "status": "ranked" if is_ranked else "experimental", "rank": rating["rank"] if rating else None, "rating": rating["rating"] if rating else None, "ciLow": rating["ci_low"] if rating else None, "ciHigh": rating["ci_high"] if rating else None, "seriesWins": rating["wins"] if rating else record["series_wins"], "seriesLosses": rating["losses"] if rating else record["series"] - record["series_wins"], "gameWins": record["game_wins"], "gameLosses": record["games"] - record["game_wins"], "calls": token_usage["calls"], "inputTokens": token_usage["input_tokens"], "cachedInputTokens": token_usage["cached_input_tokens"], "outputTokens": token_usage["output_tokens"], "reasoningTokens": token_usage["reasoning_tokens"], "totalTokens": token_usage["total_tokens"], "estimatedCostUsd": round(evaluated_cost, 4) if evaluated_cost is not None else None, "paidCalls": paid_token_usage["calls"], "paidTotalTokens": paid_token_usage["total_tokens"], "paidEstimatedCostUsd": round(paid_cost, 4) if paid_cost is not None else None, "protocols": sorted(protocols.get(model, {'unknown'})), "resolvedModels": sorted(resolved_models.get(model, set())), } ) leaderboard.sort(key=lambda row: (row["rank"] is None, row["rank"] or 999, row["name"])) ranked_games = sum(len(result.games) for result in ranked_results) experimental_games = sum(len(result.games) for result in experimental_results) total_usage, total_cost = usage_totals(paid_usage) evaluated_total_usage, _ = usage_totals(evaluated_usage) ranked_total_usage, ranked_cost = usage_totals(ranked_usage) experimental_total_usage, experimental_cost = usage_totals(experimental_usage) priced_models = {model for model in paid_usage if model in PRICES} priced_calls = sum(paid_usage[model]["calls"] for model in priced_models) priced_tokens = sum(paid_usage[model]["total_tokens"] for model in priced_models) price_rows = [] for model in models: price = PRICES.get(model) if price is None: continue price_rows.append( { "model": model, "name": DISPLAY_NAMES.get(model, model), "input": price.input_per_million, "cachedInput": price.cached_input_per_million, "output": price.output_per_million, "source": price.source, "sourceLabel": price.source_label, "note": price.note, } ) pool = json.loads((ROOT / "src" / "pokemon_bench" / "data" / "worlds_2026_masters_all.json").read_text(encoding="utf-8")) unique_teams = {team for result in ranked_results for team in result.teams.values()} unique_scenarios = { "-".join(result.match_id.split("-")[:3]) for result in ranked_results } manifests = [ json.loads((path.parent / "manifest.json").read_text(encoding="utf-8")) for path in RANKED_SUMMARIES ] manifests.extend(json.loads((directory / "manifest.json").read_text(encoding="utf-8")) for directory in system_one_directories()) source_timestamps = [ json.loads(path.read_text(encoding="utf-8"))["completed_at"] for path in RANKED_SUMMARIES ] for directory in system_one_directories(): status = json.loads((directory / "status.json").read_text(encoding="utf-8")) source_timestamps.append(status.get("updated_at") or status.get("completed_at") or status["started_at"]) evidence_updated_at = max( datetime.fromisoformat(value.replace("Z", "+00:00")) for value in source_timestamps ).isoformat().replace("+00:00", "Z") output = { "schemaVersion": "pokemon-bench-site-data-v1", "evidenceUpdatedAt": evidence_updated_at, "asOf": evidence_updated_at[:10], "leaderboard": leaderboard, "headToHead": head_to_head(ranked_results), "summary": { "rankedModels": len(ranked_models), "experimentalModels": len(experimental_models), "completedSeries": len(ranked_results) + len(experimental_results), "rankedSeries": len(ranked_results), "plannedSeries": sum(run["plannedSeries"] for run in runs), "battleGames": ranked_games + experimental_games, "rankedBattleGames": ranked_games, "usageRecords": total_usage["calls"], "inputTokens": total_usage["input_tokens"], "outputTokens": total_usage["output_tokens"], "reasoningTokens": total_usage["reasoning_tokens"], "totalTokens": total_usage["total_tokens"], "estimatedCostUsd": round(total_cost, 2), "pricedCalls": priced_calls, "unpricedCalls": total_usage["calls"] - priced_calls, "pricedTokens": priced_tokens, "unpricedTokens": total_usage["total_tokens"] - priced_tokens, "evaluatedCalls": evaluated_total_usage["calls"], "evaluatedTokens": evaluated_total_usage["total_tokens"], "teamPoolSize": len(pool["teams"]), "sampledTeams": len(unique_teams), "uniqueScenarios": len(unique_scenarios), }, "runs": runs, "pricing": price_rows, "method": { "format": "Pokémon Champions VGC 2026 Regulation M-B", "rating": "Regularized Bradley–Terry fit (MAP), transformed to an Elo-like scale centered at 1500. All models share a centered Gaussian log-strength prior with standard deviation 2 (about 347 rating points).", "unit": "One series is one best-of-three leg. Ratings use series winners, not individual game wins.", "pairing": "Each sampled team pair is played as two fresh-memory legs with models swapping teams and player seats, balancing assignment effects within that pairing.", "information": "Both models receive open team sheets, full legal-action menus, public battle state, and the opponent team before game one.", "randomness": "Team selection and each simulated battle are deterministically seeded and recorded.", "pool": f"{len(unique_teams)} of {len(pool['teams'])} reconstructed Masters teams from the 2026 World Championships appear in ranked play across {len(unique_scenarios)} paired scenarios.", "stats": pool["stat_provenance"], "sampling": "The six-model OpenAI round robin reused one team pair across all 15 pairings; other ranked pairings used seeded scenarios. Jev/Clef precision batches adaptively selected inexpensive opponents by expected posterior variance reduction, with a fixed 24-series cap and a 20% interval-width goal. Intervals remain conditional on this sampling and the prior.", "uncertainty": "The 95% Laplace posterior intervals are conditional on the Gaussian prior, observed results, and adaptive opponent selection and stopping. Legs are treated as independent despite shared mirrored teams and seeds; these intervals do not estimate uncertainty across the full Worlds team population or guarantee frequentist coverage.", "comparability": "Ratings compare evaluated systems across one connected match graph. Jev and Clef use a grouped System One decision interface rather than one joint-action JSON response; their ratings measure model-plus-interface performance. Completed series count even while a schedule is unfinished.", }, "provenance": { "promptHash": manifests[0]["battle_prompt_sha256"], "poolHash": manifests[0]["pool_sha256"], "showdownRevision": manifests[0]["engine"]["showdown_revision"], "harnessHashes": sorted({manifest["engine"]["benchmark_source_sha256"] for manifest in manifests}), "evidenceUpdatedAt": evidence_updated_at, }, "costMethod": { "scope": "Completed decisions with normalized usage in battle JSONL files. Decisions inside failed or retried match attempts remain when usage was logged; provider requests that failed before reporting usage are excluded. Leaderboard-row usage is limited to canonical trajectories for the displayed results.", "estimate": "Priced token counts multiplied by public standard API list prices as of 2026-10-02; taxes, prepaid credits, tier discounts, unlogged billing, and Clef's neuron-priced usage are excluded.", "pricing": price_rows, "usageAudit": { "canonicalFilesRead": ranked_audit["filesRead"] + experimental_audit["filesRead"], "canonicalSkippedFiles": ranked_audit["skippedFiles"] + experimental_audit["skippedFiles"], "canonicalSkippedRecords": ranked_audit["skippedRecords"] + experimental_audit["skippedRecords"], "allLogFilesRead": paid_audit["filesRead"], "bestEffortSkippedFiles": paid_audit["skippedFiles"], "bestEffortSkippedRecords": paid_audit["skippedRecords"], }, "scopes": [ {"label": "Ranked evidence", "usageRecords": ranked_total_usage["calls"], "tokens": ranked_total_usage["total_tokens"], "costUsd": round(ranked_cost, 2)}, {"label": "Retries + development", "usageRecords": total_usage["calls"] - evaluated_total_usage["calls"], "tokens": total_usage["total_tokens"] - evaluated_total_usage["total_tokens"], "costUsd": round(total_cost - ranked_cost - experimental_cost, 2)}, ], }, } OUTPUT.parent.mkdir(parents=True, exist_ok=True) OUTPUT.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") write_evidence_files() display_path = OUTPUT.relative_to(ROOT) if OUTPUT.is_relative_to(ROOT) else OUTPUT print(f"Wrote {display_path}") print(json.dumps(output["summary"], indent=2)) if __name__ == "__main__": main()