{
 "n_models": 18,
 "n_scenes": 200,
 "columns": [
  "VQA",
  "Perceptual",
  "2D Dynamics",
  "2.5D Geometry",
  "3D Geometry",
  "3D Dynamics"
 ],
 "summary_of": [
  "Perceptual",
  "2D Dynamics",
  "2.5D Geometry",
  "3D Geometry",
  "3D Dynamics"
 ],
 "family_notes": {
  "VQA": "Share of scene-specific yes/no questions a VLM answers correctly from the rendered video (all scenes).",
  "Perceptual": "How closely the rendered video resembles the reference, in DINOv3 features (all scenes).",
  "2D Dynamics": "Whether the motion on screen matches: moving-object overlap (all scenes), plus optical flow and point tracks (real videos).",
  "2.5D Geometry": "Depth error against the reference (real videos).",
  "3D Geometry": "How close the reconstructed surfaces are to the true ones (simulated scenes).",
  "3D Dynamics": "Whether matter follows its true 3D motion (simulated scenes)."
 },
 "notes": {
  "score": "Every metric is put on its declared [0,1] scale, never min-maxed over the models, so a column does not move when a model is added. A family score is the unweighted mean of its metrics; the summary score is the mean of the five reconstruction families, equal weight each.",
  "elo": "VLM Elo is Bradley-Terry over 2221 pairwise comparisons on the 200 tasks, 18 systems, displayed on the conventional 1000 anchor. Ratings normalise to the field's geometric mean, so a rating is a distance from the field's average and the whole scale moves when the field changes; only differences are comparable between two fits. Intervals are a 200-draw bootstrap over tasks.",
  "coverage": "A scene a model never rendered enters at the declared penalty rather than being dropped, so every column is over all 200 scenes.",
  "cost": "Mean US dollars and mean tokens per task over each model's 200 runs. Open-weight models are priced at OpenRouter rates.",
  "families": "VQA is reported and left out of the summary score: it is a semantic judgement rather than a reconstruction measurement."
 },
 "models": [
  {
   "run": "codex-gpt-6-astra-max",
   "tag": "astra",
   "label": "Astra [Max]",
   "vendor": "openai",
   "harness": "Codex",
   "elo": 1732.5,
   "human_elo": 1633.5,
   "families": {
    "VQA": 0.87594,
    "Perceptual": 0.90617,
    "2D Dynamics": 0.59961,
    "2.5D Geometry": 0.79113,
    "3D Geometry": 0.92221,
    "3D Dynamics": 0.7349
   },
   "overall": 0.7908,
   "vqa": 0.8759,
   "cost_usd": 11.321,
   "tokens": 6227528,
   "out_tokens": 67475,
   "steps": 66.5,
   "rendered": 200,
   "video_key": "codex-gpt-6-astra-max",
   "rank": 1,
   "human_elo_ci": [
    1570.2,
    1707.8
   ],
   "elo_ci": [
    1630.5,
    1837.6
   ],
   "by_split": {
    "real": {
     "elo": 1813.7,
     "elo_ci": [
      1640.5,
      2059.5
     ],
     "human_elo": 1702.9,
     "human_elo_ci": [
      1608.7,
      1807.4
     ],
     "vqa": 0.8533,
     "cost_usd": 11.054,
     "tokens": 6021392
    },
    "synthetic": {
     "elo": 1662.2,
     "elo_ci": [
      1539.0,
      1812.6
     ],
     "human_elo": 1565.8,
     "human_elo_ci": [
      1470.4,
      1666.9
     ],
     "vqa": 0.8983,
     "cost_usd": 11.588,
     "tokens": 6433665
    }
   }
  },
  {
   "run": "claude-claude-opus-5-5-high",
   "tag": "opus55",
   "label": "Opus 5.5 [High]",
   "vendor": "anthropic",
   "harness": "Claude Code",
   "elo": 1556.0,
   "human_elo": null,
   "families": {
    "VQA": 0.82755,
    "Perceptual": 0.88214,
    "2D Dynamics": 0.55597,
    "2.5D Geometry": 0.77099,
    "3D Geometry": 0.93709,
    "3D Dynamics": 0.74586
   },
   "overall": 0.7784,
   "vqa": 0.8276,
   "cost_usd": 6.269,
   "tokens": 12366054,
   "out_tokens": 113408,
   "steps": 90.5,
   "rendered": 200,
   "video_key": "claude-claude-opus-5-5-high",
   "rank": 2,
   "human_elo_ci": null,
   "elo_ci": [
    1474.5,
    1656.2
   ],
   "by_split": {
    "real": {
     "elo": 1480.9,
     "elo_ci": [
      1371.9,
      1638.8
     ],
     "human_elo": null,
     "human_elo_ci": null,
     "vqa": 0.7751,
     "cost_usd": 5.457,
     "tokens": 9983377
    },
    "synthetic": {
     "elo": 1612.1,
     "elo_ci": [
      1506.2,
      1775.2
     ],
     "human_elo": null,
     "human_elo_ci": null,
     "vqa": 0.8795,
     "cost_usd": 7.081,
     "tokens": 14748730
    }
   }
  },
  {
   "run": "codex-gpt-6-astra-high",
   "tag": "astrahigh",
   "label": "Astra [High]",
   "vendor": "openai",
   "harness": "Codex",
   "elo": 1685.1,
   "human_elo": 1586.1,
   "families": {
    "VQA": 0.85014,
    "Perceptual": 0.89724,
    "2D Dynamics": 0.55136,
    "2.5D Geometry": 0.77013,
    "3D Geometry": 0.91912,
    "3D Dynamics": 0.68866
   },
   "overall": 0.7653,
   "vqa": 0.8501,
   "cost_usd": 6.279,
   "tokens": 3921016,
   "out_tokens": 29279,
   "steps": 66.2,
   "rendered": 200,
   "video_key": "codex-gpt-6-astra-high",
   "rank": 3,
   "human_elo_ci": [
    1512.9,
    1649.0
   ],
   "elo_ci": [
    1585.8,
    1811.1
   ],
   "by_split": {
    "real": {
     "elo": 1680.6,
     "elo_ci": [
      1559.5,
      1818.7
     ],
     "human_elo": 1644.8,
     "human_elo_ci": [
      1548.0,
      1739.8
     ],
     "vqa": 0.8279,
     "cost_usd": 6.092,
     "tokens": 3745385
    },
    "synthetic": {
     "elo": 1683.6,
     "elo_ci": [
      1524.8,
      1821.0
     ],
     "human_elo": 1532.6,
     "human_elo_ci": [
      1450.0,
      1615.4
     ],
     "vqa": 0.8722,
     "cost_usd": 6.465,
     "tokens": 4096648
    }
   }
  },
  {
   "run": "claude-claude-fable-5-1-high",
   "tag": "fable",
   "label": "Fable 5.1 [High]",
   "vendor": "anthropic",
   "harness": "Claude Code",
   "elo": 1391.4,
   "human_elo": 1393.9,
   "families": {
    "VQA": 0.78845,
    "Perceptual": 0.85829,
    "2D Dynamics": 0.51822,
    "2.5D Geometry": 0.74193,
    "3D Geometry": 0.90167,
    "3D Dynamics": 0.68496
   },
   "overall": 0.741,
   "vqa": 0.7884,
   "cost_usd": 15.074,
   "tokens": 11978563,
   "out_tokens": 140433,
   "steps": 79.4,
   "rendered": 200,
   "video_key": "claude-claude-fable-5-1-high",
   "rank": 4,
   "human_elo_ci": [
    1349.3,
    1446.3
   ],
   "elo_ci": [
    1333.4,
    1459.9
   ],
   "by_split": {
    "real": {
     "elo": 1357.8,
     "elo_ci": [
      1230.4,
      1478.5
     ],
     "human_elo": 1443.4,
     "human_elo_ci": [
      1365.5,
      1523.3
     ],
     "vqa": 0.7524,
     "cost_usd": 12.757,
     "tokens": 8305608
    },
    "synthetic": {
     "elo": 1391.9,
     "elo_ci": [
      1312.0,
      1488.2
     ],
     "human_elo": 1367.9,
     "human_elo_ci": [
      1300.1,
      1432.3
     ],
     "vqa": 0.8242,
     "cost_usd": 17.39,
     "tokens": 15651517
    }
   }
  },
  {
   "run": "codex-gpt-6-astra-low",
   "tag": "astralow",
   "label": "Astra [Low]",
   "vendor": "openai",
   "harness": "Codex",
   "elo": 1432.6,
   "human_elo": 1393.3,
   "families": {
    "VQA": 0.7824,
    "Perceptual": 0.878,
    "2D Dynamics": 0.47869,
    "2.5D Geometry": 0.75085,
    "3D Geometry": 0.89795,
    "3D Dynamics": 0.62643
   },
   "overall": 0.7264,
   "vqa": 0.7824,
   "cost_usd": 3.495,
   "tokens": 2251703,
   "out_tokens": 13439,
   "steps": 55.3,
   "rendered": 200,
   "video_key": "codex-gpt-6-astra-low",
   "rank": 5,
   "human_elo_ci": [
    1347.5,
    1436.1
   ],
   "elo_ci": [
    1368.1,
    1499.3
   ],
   "by_split": {
    "real": {
     "elo": 1452.5,
     "elo_ci": [
      1376.4,
      1550.4
     ],
     "human_elo": 1439.9,
     "human_elo_ci": [
      1369.8,
      1505.6
     ],
     "vqa": 0.7518,
     "cost_usd": 3.469,
     "tokens": 2206408
    },
    "synthetic": {
     "elo": 1397.6,
     "elo_ci": [
      1307.3,
      1489.6
     ],
     "human_elo": 1366.5,
     "human_elo_ci": [
      1305.8,
      1427.0
     ],
     "vqa": 0.8127,
     "cost_usd": 3.521,
     "tokens": 2296999
    }
   }
  },
  {
   "run": "claude-claude-opus-5-high",
   "tag": "opus",
   "label": "Opus 5 [High]",
   "vendor": "anthropic",
   "harness": "Claude Code",
   "elo": 1360.8,
   "human_elo": 1388.5,
   "families": {
    "VQA": 0.76424,
    "Perceptual": 0.85094,
    "2D Dynamics": 0.50632,
    "2.5D Geometry": 0.73178,
    "3D Geometry": 0.86078,
    "3D Dynamics": 0.63638
   },
   "overall": 0.7172,
   "vqa": 0.7642,
   "cost_usd": 17.52,
   "tokens": 18204136,
   "out_tokens": 253626,
   "steps": 75.6,
   "rendered": 200,
   "video_key": "claude-claude-opus-5-high",
   "rank": 6,
   "human_elo_ci": [
    1345.1,
    1434.4
   ],
   "elo_ci": [
    1300.6,
    1441.8
   ],
   "by_split": {
    "real": {
     "elo": 1322.8,
     "elo_ci": [
      1222.0,
      1416.5
     ],
     "human_elo": 1402.9,
     "human_elo_ci": [
      1327.1,
      1474.9
     ],
     "vqa": 0.7172,
     "cost_usd": 15.443,
     "tokens": 15643536
    },
    "synthetic": {
     "elo": 1375.5,
     "elo_ci": [
      1286.4,
      1473.5
     ],
     "human_elo": 1375.1,
     "human_elo_ci": [
      1318.7,
      1437.4
     ],
     "vqa": 0.8108,
     "cost_usd": 19.598,
     "tokens": 20764735
    }
   }
  },
  {
   "run": "codex-gpt-5.6-luna-max",
   "tag": "luna",
   "label": "Luna [Max]",
   "vendor": "openai",
   "harness": "Codex",
   "elo": 1223.7,
   "human_elo": 1071.6,
   "families": {
    "VQA": 0.6731,
    "Perceptual": 0.83376,
    "2D Dynamics": 0.41967,
    "2.5D Geometry": 0.70013,
    "3D Geometry": 0.86128,
    "3D Dynamics": 0.53254
   },
   "overall": 0.6695,
   "vqa": 0.6731,
   "cost_usd": 0.673,
   "tokens": 23278221,
   "out_tokens": 106404,
   "steps": 191.1,
   "rendered": 200,
   "video_key": "codex-gpt-5.6-luna-max",
   "rank": 7,
   "human_elo_ci": [
    1020.8,
    1111.3
   ],
   "elo_ci": [
    1162.5,
    1287.6
   ],
   "by_split": {
    "real": {
     "elo": 1251.3,
     "elo_ci": [
      1156.6,
      1343.7
     ],
     "human_elo": 1111.1,
     "human_elo_ci": [
      1043.7,
      1194.5
     ],
     "vqa": 0.6225,
     "cost_usd": 0.588,
     "tokens": 20117195
    },
    "synthetic": {
     "elo": 1188.4,
     "elo_ci": [
      1099.5,
      1285.5
     ],
     "human_elo": 1034.6,
     "human_elo_ci": [
      974.9,
      1088.6
     ],
     "vqa": 0.7232,
     "cost_usd": 0.759,
     "tokens": 26439246
    }
   }
  },
  {
   "run": "codex-gpt-5.6-sol-high",
   "tag": "sol",
   "label": "Sol [High]",
   "vendor": "openai",
   "harness": "Codex",
   "elo": 1171.1,
   "human_elo": 1118.6,
   "families": {
    "VQA": 0.67431,
    "Perceptual": 0.83343,
    "2D Dynamics": 0.35659,
    "2.5D Geometry": 0.63128,
    "3D Geometry": 0.8455,
    "3D Dynamics": 0.56032
   },
   "overall": 0.6454,
   "vqa": 0.6743,
   "cost_usd": 2.471,
   "tokens": 2603171,
   "out_tokens": 26032,
   "steps": 51.0,
   "rendered": 200,
   "video_key": "codex-gpt-5.6-sol-high",
   "rank": 8,
   "human_elo_ci": [
    1079.7,
    1154.4
   ],
   "elo_ci": [
    1105.1,
    1245.7
   ],
   "by_split": {
    "real": {
     "elo": 1130.8,
     "elo_ci": [
      1029.4,
      1234.9
     ],
     "human_elo": 1109.6,
     "human_elo_ci": [
      1056.9,
      1156.0
     ],
     "vqa": 0.6341,
     "cost_usd": 2.245,
     "tokens": 2267877
    },
    "synthetic": {
     "elo": 1196.0,
     "elo_ci": [
      1105.6,
      1299.4
     ],
     "human_elo": 1124.1,
     "human_elo_ci": [
      1076.6,
      1170.3
     ],
     "vqa": 0.7141,
     "cost_usd": 2.697,
     "tokens": 2938465
    }
   }
  },
  {
   "run": "antigravity-gemini-3.8-flash-high",
   "tag": "geminif",
   "label": "Gemini 3.8 [High]",
   "vendor": "google",
   "harness": "Antigravity",
   "elo": 1037.3,
   "human_elo": 1071.7,
   "families": {
    "VQA": 0.62651,
    "Perceptual": 0.80942,
    "2D Dynamics": 0.36298,
    "2.5D Geometry": 0.62684,
    "3D Geometry": 0.84046,
    "3D Dynamics": 0.55726
   },
   "overall": 0.6394,
   "vqa": 0.6265,
   "cost_usd": 2.721,
   "tokens": 35974229,
   "out_tokens": 206201,
   "steps": 292.7,
   "rendered": 200,
   "video_key": "antigravity-gemini-3.8-flash-high",
   "rank": 9,
   "human_elo_ci": [
    1035.6,
    1110.3
   ],
   "elo_ci": [
    970.2,
    1108.5
   ],
   "by_split": {
    "real": {
     "elo": 988.0,
     "elo_ci": [
      894.5,
      1075.7
     ],
     "human_elo": 980.0,
     "human_elo_ci": [
      912.3,
      1036.3
     ],
     "vqa": 0.5089,
     "cost_usd": 2.448,
     "tokens": 29865134
    },
    "synthetic": {
     "elo": 1080.8,
     "elo_ci": [
      994.3,
      1178.7
     ],
     "human_elo": 1132.4,
     "human_elo_ci": [
      1085.9,
      1183.7
     ],
     "vqa": 0.7429,
     "cost_usd": 2.998,
     "tokens": 42208000
    }
   }
  },
  {
   "run": "stirrup-deepseek-v4.1-flash-default",
   "tag": "deepseek",
   "label": "DeepSeek V4.1",
   "vendor": "deepseek",
   "harness": "Stirrup",
   "elo": 886.0,
   "human_elo": 912.5,
   "families": {
    "VQA": 0.50029,
    "Perceptual": 0.77898,
    "2D Dynamics": 0.338,
    "2.5D Geometry": 0.61722,
    "3D Geometry": 0.76942,
    "3D Dynamics": 0.47873
   },
   "overall": 0.5965,
   "vqa": 0.5003,
   "cost_usd": 0.442,
   "tokens": 32048244,
   "out_tokens": 134310,
   "steps": 257.8,
   "rendered": 199,
   "video_key": "stirrup-deepseek-v4.1-flash-default",
   "rank": 10,
   "human_elo_ci": [
    869.7,
    948.4
   ],
   "elo_ci": [
    812.1,
    942.4
   ],
   "by_split": {
    "real": {
     "elo": 847.4,
     "elo_ci": [
      759.2,
      923.5
     ],
     "human_elo": 881.6,
     "human_elo_ci": [
      824.2,
      940.9
     ],
     "vqa": 0.4057,
     "cost_usd": 0.397,
     "tokens": 28484534
    },
    "synthetic": {
     "elo": 929.4,
     "elo_ci": [
      835.0,
      1031.2
     ],
     "human_elo": 944.1,
     "human_elo_ci": [
      882.1,
      1017.2
     ],
     "vqa": 0.5939,
     "cost_usd": 0.487,
     "tokens": 35611954
    }
   }
  },
  {
   "run": "stirrup-qwen3.8-flash-next-default",
   "tag": "qwen",
   "label": "Qwen3.8",
   "vendor": "alibaba",
   "harness": "Stirrup",
   "elo": 1061.1,
   "human_elo": 1091.7,
   "families": {
    "VQA": 0.58012,
    "Perceptual": 0.7073,
    "2D Dynamics": 0.34349,
    "2.5D Geometry": 0.55364,
    "3D Geometry": 0.71359,
    "3D Dynamics": 0.4955
   },
   "overall": 0.5627,
   "vqa": 0.5801,
   "cost_usd": 2.771,
   "tokens": 69246813,
   "out_tokens": 1505896,
   "steps": 752.4,
   "rendered": 181,
   "video_key": "stirrup-qwen3.8-flash-next-default",
   "rank": 11,
   "human_elo_ci": [
    1047.2,
    1135.9
   ],
   "elo_ci": [
    992.6,
    1126.2
   ],
   "by_split": {
    "real": {
     "elo": 1062.4,
     "elo_ci": [
      961.2,
      1166.6
     ],
     "human_elo": 1004.7,
     "human_elo_ci": [
      945.3,
      1059.8
     ],
     "vqa": 0.4866,
     "cost_usd": 2.704,
     "tokens": 68209015
    },
    "synthetic": {
     "elo": 1049.5,
     "elo_ci": [
      947.5,
      1149.9
     ],
     "human_elo": 1161.0,
     "human_elo_ci": [
      1109.1,
      1214.5
     ],
     "vqa": 0.6727,
     "cost_usd": 2.837,
     "tokens": 70284611
    }
   }
  },
  {
   "run": "codex-gpt-5.6-terra-high",
   "tag": "terra",
   "label": "Terra [High]",
   "vendor": "openai",
   "harness": "Codex",
   "elo": 911.1,
   "human_elo": 916.9,
   "families": {
    "VQA": 0.4843,
    "Perceptual": 0.79399,
    "2D Dynamics": 0.22905,
    "2.5D Geometry": 0.52294,
    "3D Geometry": 0.77157,
    "3D Dynamics": 0.43488
   },
   "overall": 0.5505,
   "vqa": 0.4843,
   "cost_usd": 0.621,
   "tokens": 1448734,
   "out_tokens": 17283,
   "steps": 37.5,
   "rendered": 200,
   "video_key": "codex-gpt-5.6-terra-high",
   "rank": 12,
   "human_elo_ci": [
    870.8,
    956.2
   ],
   "elo_ci": [
    847.8,
    969.5
   ],
   "by_split": {
    "real": {
     "elo": 910.0,
     "elo_ci": [
      821.7,
      1007.1
     ],
     "human_elo": 945.4,
     "human_elo_ci": [
      896.5,
      997.2
     ],
     "vqa": 0.4082,
     "cost_usd": 0.575,
     "tokens": 1285398
    },
    "synthetic": {
     "elo": 906.1,
     "elo_ci": [
      791.2,
      1023.0
     ],
     "human_elo": 891.0,
     "human_elo_ci": [
      840.6,
      942.6
     ],
     "vqa": 0.5596,
     "cost_usd": 0.667,
     "tokens": 1612070
    }
   }
  },
  {
   "run": "stirrup-glm-5.3-flash-default",
   "tag": "glm",
   "label": "GLM 5.3",
   "vendor": "zhipu",
   "harness": "Stirrup",
   "elo": 669.8,
   "human_elo": 732.5,
   "families": {
    "VQA": 0.29528,
    "Perceptual": 0.69128,
    "2D Dynamics": 0.19626,
    "2.5D Geometry": 0.51355,
    "3D Geometry": 0.60062,
    "3D Dynamics": 0.36448
   },
   "overall": 0.4732,
   "vqa": 0.2953,
   "cost_usd": 6.278,
   "tokens": 166449595,
   "out_tokens": 451636,
   "steps": 450.8,
   "rendered": 196,
   "video_key": "stirrup-glm-5.3-flash-default",
   "rank": 13,
   "human_elo_ci": [
    680.5,
    779.8
   ],
   "elo_ci": [
    567.3,
    767.5
   ],
   "by_split": {
    "real": {
     "elo": 733.8,
     "elo_ci": [
      604.6,
      840.4
     ],
     "human_elo": 711.6,
     "human_elo_ci": [
      648.0,
      774.6
     ],
     "vqa": 0.2986,
     "cost_usd": 6.33,
     "tokens": 167848526
    },
    "synthetic": {
     "elo": 625.0,
     "elo_ci": [
      498.9,
      750.4
     ],
     "human_elo": 749.7,
     "human_elo_ci": [
      682.5,
      811.8
     ],
     "vqa": 0.292,
     "cost_usd": 6.226,
     "tokens": 165050664
    }
   }
  },
  {
   "run": "stirrup-muse-glimmer-high",
   "tag": "muse",
   "label": "Muse [High]",
   "vendor": "muse",
   "harness": "Stirrup",
   "elo": 345.1,
   "human_elo": 504.6,
   "families": {
    "VQA": 0.10957,
    "Perceptual": 0.61123,
    "2D Dynamics": 0.11254,
    "2.5D Geometry": 0.47331,
    "3D Geometry": 0.68392,
    "3D Dynamics": 0.32825
   },
   "overall": 0.4419,
   "vqa": 0.1096,
   "cost_usd": 0.288,
   "tokens": 4052501,
   "out_tokens": 45242,
   "steps": 95.9,
   "rendered": 199,
   "video_key": "stirrup-muse-glimmer-high",
   "rank": 14,
   "human_elo_ci": [
    458.3,
    555.4
   ],
   "elo_ci": [
    238.3,
    417.7
   ],
   "by_split": {
    "real": {
     "elo": 424.0,
     "elo_ci": [
      280.7,
      541.9
     ],
     "human_elo": 475.6,
     "human_elo_ci": [
      388.6,
      560.2
     ],
     "vqa": 0.0676,
     "cost_usd": 0.229,
     "tokens": 3133331
    },
    "synthetic": {
     "elo": 301.9,
     "elo_ci": [
      170.7,
      423.8
     ],
     "human_elo": 536.1,
     "human_elo_ci": [
      467.7,
      588.0
     ],
     "vqa": 0.1511,
     "cost_usd": 0.346,
     "tokens": 4971671
    }
   }
  },
  {
   "run": "stirrup-mimo-v2.5-default",
   "tag": "mimo",
   "label": "MiMo V2.5",
   "vendor": "xiaomi",
   "harness": "Stirrup",
   "elo": 568.9,
   "human_elo": 675.6,
   "families": {
    "VQA": 0.24636,
    "Perceptual": 0.68953,
    "2D Dynamics": 0.11561,
    "2.5D Geometry": 0.48298,
    "3D Geometry": 0.57108,
    "3D Dynamics": 0.31459
   },
   "overall": 0.4348,
   "vqa": 0.2464,
   "cost_usd": 0.163,
   "tokens": 9871911,
   "out_tokens": 153926,
   "steps": 80.8,
   "rendered": 196,
   "video_key": "stirrup-mimo-v2.5-default",
   "rank": 15,
   "human_elo_ci": [
    634.6,
    717.3
   ],
   "elo_ci": [
    469.6,
    639.8
   ],
   "by_split": {
    "real": {
     "elo": 579.3,
     "elo_ci": [
      459.6,
      694.2
     ],
     "human_elo": 667.2,
     "human_elo_ci": [
      610.4,
      714.6
     ],
     "vqa": 0.1826,
     "cost_usd": 0.159,
     "tokens": 9447256
    },
    "synthetic": {
     "elo": 581.4,
     "elo_ci": [
      453.3,
      685.2
     ],
     "human_elo": 676.8,
     "human_elo_ci": [
      604.8,
      751.8
     ],
     "vqa": 0.3095,
     "cost_usd": 0.167,
     "tokens": 10296567
    }
   }
  },
  {
   "run": "stirrup-minimax-m3-default",
   "tag": "minimax",
   "label": "MiniMax M3",
   "vendor": "minimax",
   "harness": "Stirrup",
   "elo": 554.3,
   "human_elo": 725.4,
   "families": {
    "VQA": 0.17852,
    "Perceptual": 0.5247,
    "2D Dynamics": 0.11381,
    "2.5D Geometry": 0.35493,
    "3D Geometry": 0.44452,
    "3D Dynamics": 0.25882
   },
   "overall": 0.3394,
   "vqa": 0.1785,
   "cost_usd": 2.472,
   "tokens": 31271178,
   "out_tokens": 144665,
   "steps": 244.3,
   "rendered": 160,
   "video_key": "stirrup-minimax-m3-default",
   "rank": 16,
   "human_elo_ci": [
    679.3,
    768.1
   ],
   "elo_ci": [
    458.1,
    627.9
   ],
   "by_split": {
    "real": {
     "elo": 588.7,
     "elo_ci": [
      491.5,
      689.8
     ],
     "human_elo": 718.6,
     "human_elo_ci": [
      665.7,
      774.3
     ],
     "vqa": 0.1445,
     "cost_usd": 2.465,
     "tokens": 31156577
    },
    "synthetic": {
     "elo": 540.4,
     "elo_ci": [
      411.7,
      672.6
     ],
     "human_elo": 721.1,
     "human_elo_ci": [
      661.3,
      789.1
     ],
     "vqa": 0.2122,
     "cost_usd": 2.479,
     "tokens": 31385779
    }
   }
  },
  {
   "run": "stirrup-mistral-medium-3.5-default",
   "tag": "mistral",
   "label": "Mistral 3.5",
   "vendor": "mistral",
   "harness": "Stirrup",
   "elo": 138.0,
   "human_elo": 361.2,
   "families": {
    "VQA": 0.01278,
    "Perceptual": 0.46069,
    "2D Dynamics": 0.02382,
    "2.5D Geometry": 0.411,
    "3D Geometry": 0.42193,
    "3D Dynamics": 0.25858
   },
   "overall": 0.3152,
   "vqa": 0.0128,
   "cost_usd": 3.568,
   "tokens": 2232616,
   "out_tokens": 36589,
   "steps": 62.8,
   "rendered": 183,
   "video_key": "stirrup-mistral-medium-3.5-default",
   "rank": 17,
   "human_elo_ci": [
    309.8,
    414.8
   ],
   "elo_ci": [
    1.3,
    249.0
   ],
   "by_split": {
    "real": {
     "elo": 137.7,
     "elo_ci": [
      -135.5,
      313.1
     ],
     "human_elo": 336.1,
     "human_elo_ci": [
      245.2,
      409.6
     ],
     "vqa": 0.0067,
     "cost_usd": 3.508,
     "tokens": 2196149
    },
    "synthetic": {
     "elo": 160.3,
     "elo_ci": [
      -3.2,
      271.8
     ],
     "human_elo": 399.0,
     "human_elo_ci": [
      303.0,
      477.9
     ],
     "vqa": 0.0188,
     "cost_usd": 3.628,
     "tokens": 2269083
    }
   }
  },
  {
   "run": "stirrup-gemma-4-31b-default",
   "tag": "gemma",
   "label": "Gemma 4 31B",
   "vendor": "google",
   "harness": "Stirrup",
   "elo": 275.3,
   "human_elo": 422.2,
   "families": {
    "VQA": 0.05645,
    "Perceptual": 0.51207,
    "2D Dynamics": 0.00436,
    "2.5D Geometry": 0.13196,
    "3D Geometry": 0.1663,
    "3D Dynamics": 0.12604
   },
   "overall": 0.1881,
   "vqa": 0.0564,
   "cost_usd": 0.078,
   "tokens": 1332711,
   "out_tokens": 19330,
   "steps": 32.3,
   "rendered": 200,
   "video_key": "stirrup-gemma-4-31b-default",
   "rank": 18,
   "human_elo_ci": [
    364.2,
    475.2
   ],
   "elo_ci": [
    172.1,
    365.6
   ],
   "by_split": {
    "real": {
     "elo": 238.2,
     "elo_ci": [
      102.9,
      351.3
     ],
     "human_elo": 424.6,
     "human_elo_ci": [
      344.5,
      499.5
     ],
     "vqa": 0.0327,
     "cost_usd": 0.071,
     "tokens": 1204864
    },
    "synthetic": {
     "elo": 317.8,
     "elo_ci": [
      189.5,
      436.0
     ],
     "human_elo": 422.2,
     "human_elo_ci": [
      346.2,
      487.9
     ],
     "vqa": 0.08,
     "cost_usd": 0.086,
     "tokens": 1460557
    }
   }
  }
 ]
}