{
 "benchmark": "Ad Writing Arena",
 "updated": "2026-11-01",
 "status": "complete",
 "summary": "AI models write a short video ad script from the same brief. A fixed panel of LLM judges compares the scripts in blind pairs, in both orders. A Bradley–Terry model turns the verdicts into ratings. The product is a real car perfume; its brand name and trademarks are changed to a fictional brand, Noir Drive.",
 "methodology": "https://convictionlabs.ai/arena/methodology.md",
 "data": "https://convictionlabs.ai/arena/data.json",
 "judges": [
  "opus-5.5",
  "fable-5.1",
  "gpt-6-astra",
  "gpt-6.1-sol",
  "gemini-4-argon",
  "muse-wonder",
  "kimi-k3.1-pro",
  "glm-5.3-prime",
  "grok-4.7"
 ],
 "judgments": 1631,
 "order_consistency": 0.83,
 "invalid_judgments_excluded": 3,
 "definitions": {
  "rating": "Bradley-Terry strength on an Elo-like scale, 1000 = average model; ties count as one win each way.",
  "ci95": "2.5th-97.5th percentile over 200 bootstrap samples of script pairs.",
  "top5_chance": "share of 500 bootstrap samples of script pairs in which the model ranks in the top five.",
  "win_rate": "wins / judgments; tie = half a win; neither = a loss for both.",
  "constraints_met": "share of judge readings where the script met all four hard constraints.",
  "exact_length": "briefs where the spoken word count is inside the target range.",
  "cost_per_script_usd": "API list price; null for subscription calls."
 },
 "models": [
  {
   "model": "opus-5.5",
   "label": "Claude Opus 5.5",
   "lab": "Anthropic",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": true,
   "rating": 1365,
   "ci95": [
    1316,
    1455
   ],
   "win_rate": 0.786,
   "constraints_met": 0.816,
   "exact_length": "3/3",
   "cost_per_script_usd": 0.0486,
   "seconds_per_script": 16,
   "judgments": 234,
   "top5_chance": 1.0,
   "rank": 1
  },
  {
   "model": "muse-wonder",
   "label": "Muse Wonder",
   "lab": "Meta",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": true,
   "rating": 1224,
   "ci95": [
    1136,
    1316
   ],
   "win_rate": 0.668,
   "constraints_met": 0.828,
   "exact_length": "3/3",
   "cost_per_script_usd": 0.038,
   "seconds_per_script": 10,
   "judgments": 232,
   "top5_chance": 0.96,
   "rank": 2
  },
  {
   "model": "kimi-k3.1-pro",
   "label": "Kimi K3.1 Pro",
   "lab": "Moonshot AI",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": false,
   "rating": 1210,
   "ci95": [
    1140,
    1288
   ],
   "win_rate": 0.628,
   "constraints_met": 0.795,
   "exact_length": "3/3",
   "cost_per_script_usd": 0.0328,
   "seconds_per_script": 71,
   "judgments": 234,
   "top5_chance": 0.962,
   "rank": 3
  },
  {
   "model": "grok-4.7",
   "label": "Grok 4.7",
   "lab": "xAI",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": false,
   "rating": 1191,
   "ci95": [
    1122,
    1294
   ],
   "win_rate": 0.677,
   "constraints_met": 0.948,
   "exact_length": "2/3",
   "cost_per_script_usd": 0.019,
   "seconds_per_script": 22,
   "judgments": 232,
   "top5_chance": 0.864,
   "rank": 4
  },
  {
   "model": "mimo-v2.6-pro",
   "label": "MiMo V2.6 Pro",
   "lab": "Xiaomi",
   "setting": "low reasoning",
   "role": "writer",
   "added_later": true,
   "rating": 1142,
   "ci95": [
    1064,
    1226
   ],
   "win_rate": 0.476,
   "constraints_met": 0.541,
   "exact_length": "1/3",
   "cost_per_script_usd": 0.003,
   "seconds_per_script": 41,
   "judgments": 231,
   "top5_chance": 0.54,
   "rank": 5
  },
  {
   "model": "deepseek-v4-pro",
   "label": "DeepSeek V4 Pro",
   "lab": "DeepSeek",
   "setting": "low reasoning",
   "role": "writer",
   "added_later": false,
   "rating": 1139,
   "ci95": [
    1049,
    1250
   ],
   "win_rate": 0.588,
   "constraints_met": 0.79,
   "exact_length": "3/3",
   "cost_per_script_usd": 0.0063,
   "seconds_per_script": 43,
   "judgments": 233,
   "top5_chance": 0.504,
   "rank": 6
  },
  {
   "model": "gpt-6-astra",
   "label": "GPT-6 Astra",
   "lab": "OpenAI",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": true,
   "rating": 1099,
   "ci95": [
    1039,
    1177
   ],
   "win_rate": 0.599,
   "constraints_met": 1.0,
   "exact_length": "3/3",
   "cost_per_script_usd": null,
   "seconds_per_script": 31,
   "judgments": 232,
   "top5_chance": 0.13,
   "rank": 7
  },
  {
   "model": "gpt-6.1-sol",
   "label": "GPT-6.1 Sol",
   "lab": "OpenAI",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": false,
   "rating": 1072,
   "ci95": [
    1005,
    1146
   ],
   "win_rate": 0.558,
   "constraints_met": 0.953,
   "exact_length": "3/3",
   "cost_per_script_usd": null,
   "seconds_per_script": 22,
   "judgments": 233,
   "top5_chance": 0.036,
   "rank": 8
  },
  {
   "model": "gemini-4-argon",
   "label": "Gemini 4 Argon",
   "lab": "Google",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": true,
   "rating": 995,
   "ci95": [
    882,
    1099
   ],
   "win_rate": 0.274,
   "constraints_met": 0.333,
   "exact_length": "0/3",
   "cost_per_script_usd": 0.0123,
   "seconds_per_script": 11,
   "judgments": 234,
   "top5_chance": 0.004,
   "rank": 9
  },
  {
   "model": "glm-5.3-prime",
   "label": "GLM-5.3 Prime",
   "lab": "Z.ai",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": false,
   "rating": 917,
   "ci95": [
    805,
    1020
   ],
   "win_rate": 0.193,
   "constraints_met": 0.236,
   "exact_length": "1/3",
   "cost_per_script_usd": 0.0124,
   "seconds_per_script": 7,
   "judgments": 233,
   "top5_chance": 0.0,
   "rank": 10
  },
  {
   "model": "qwen3.8-max",
   "label": "Qwen 3.8 Max",
   "lab": "Alibaba",
   "setting": "low reasoning",
   "role": "writer",
   "added_later": false,
   "rating": 728,
   "ci95": [
    640,
    790
   ],
   "win_rate": 0.09,
   "constraints_met": 0.129,
   "exact_length": "1/3",
   "cost_per_script_usd": 0.0472,
   "seconds_per_script": 31,
   "judgments": 233,
   "top5_chance": 0.0,
   "rank": 11
  },
  {
   "model": "fable-5.1",
   "label": "Claude Fable 5.1",
   "lab": "Anthropic",
   "setting": "low reasoning",
   "role": "writer and judge",
   "added_later": false,
   "rating": 725,
   "ci95": [
    597,
    824
   ],
   "win_rate": 0.06,
   "constraints_met": 0.064,
   "exact_length": "0/3",
   "cost_per_script_usd": 0.0925,
   "seconds_per_script": 14,
   "judgments": 234,
   "top5_chance": 0.0,
   "rank": 12
  },
  {
   "model": "llama-3.1-70b",
   "label": "Llama 3.1 70B",
   "lab": "Meta, 2024",
   "setting": "no reasoning",
   "role": "anchor",
   "added_later": false,
   "rating": 697,
   "ci95": [
    584,
    778
   ],
   "win_rate": 0.081,
   "constraints_met": 0.235,
   "exact_length": "1/3",
   "cost_per_script_usd": 0.0019,
   "seconds_per_script": 9,
   "judgments": 234,
   "top5_chance": 0.0,
   "rank": 13
  },
  {
   "model": "mistral-small-3",
   "label": "Mistral Small 3",
   "lab": "Mistral, early 2025",
   "setting": "no reasoning",
   "role": "anchor",
   "added_later": false,
   "rating": 497,
   "ci95": [
    375,
    571
   ],
   "win_rate": 0.004,
   "constraints_met": 0.043,
   "exact_length": "3/3",
   "cost_per_script_usd": 0.0002,
   "seconds_per_script": 13,
   "judgments": 233,
   "top5_chance": 0.0,
   "rank": 14
  }
 ],
 "round1": {
  "judges": [
   "fable-5.1",
   "gpt-6.1-sol",
   "grok-4.7",
   "deepseek-v4-pro",
   "qwen3.8-max",
   "kimi-k3.1-pro",
   "glm-5.3-prime"
  ],
  "judgments": 1629,
  "kendall_tau_with_current": 0.934,
  "data": "https://convictionlabs.ai/arena/round1.json",
  "note": "an earlier panel that let a judge compare scripts from its own lab; replaced"
 }
}