Evaluating Multimodal Vision Models with Moonshot PerceptionBench Using Robust Data Loading and Automated Judging
def run_eval(records, backend):
print(f”\n[eval] backend = {backend.name} on {len(records)} questions”)
t0 = time.time()
preds = backend.predict_batch(records)
rows = []
for rec, raw in zip(records, preds):
if CFG[“JUDGE”] == “llm” and CFG[“API_KEY”]:
ok, how = llm_judge(raw, rec[“answer”], rec[“problem”])
else:
ok, how = rule_judge(raw, rec[“answer”], CFG[“NUM_REL_TOL”])
rows.append(dict(index=rec[“index”], code=rec[“code”], category=rec[“category”],
source_bmk=rec[“source_bmk”], n_images=rec[“n_images”],
q_chars=rec[“q_chars”], max_side=rec[“max_side”],
ans_type=answer_type(rec[“answer”]),
gold=rec[“answer”], pred=extract_answer(raw),
raw=str(raw)[:2000], correct=ok, how=how))
print(f”[eval] done in {time.time()-t0:.1f}s”)
return pd.DataFrame(rows)
def bootstrap_ci(vals, n_boot=4000, seed=0):
a = np.asarray(vals, dtype=float)
if a.size == 0:
return (float(“nan”), float(“nan”))
rng = np.random.default_rng(seed)
means = a[rng.integers(0, a.size, (n_boot, a.size))].mean(axis=1)
return tuple(np.percentile(means, [2.5, 97.5]) * 100)
def report(res, label):
print(“\n” + “=” * 78)
print(f”§9 RESULTS — {label}”)
print(“=” * 78)
lo, hi = bootstrap_ci(res.correct)
print(f”\nOVERALL accuracy: {res.correct.mean()*100:5.1f}% ”
f”95% CI [{lo:.1f}, {hi:.1f}] (n={len(res)})”)
print(“(card: no frontier model exceeds 60% overall)\n”)
print(“– per atomic capability –“)
tab = []
for code, g in res.groupby(“code”):
l, h = bootstrap_ci(g.correct)
tab.append(dict(code=code, capability=CODE_FULL.get(code, code),
n=len(g), acc=g.correct.mean() * 100, lo=l, hi=h))
t = pd.DataFrame(tab).sort_values(“acc”, ascending=False)
print(t.to_string(index=False, float_format=lambda x: f”{x:6.1f}”))
print(“\n– difficulty slices –“)
res = res.copy()
res[“img_bucket”] = np.where(res.n_images > 1, “multi-image”, “single-image”)
res[“res_bucket”] = pd.cut(res.max_side, [0, 800, 1600, 10**6],
labels=[“<800px”, “800-1600px”, “>1600px”])
for col in [“img_bucket”, “res_bucket”, “ans_type”]:
s = res.groupby(col, observed=True).correct.agg([“size”, “mean”])
s[“mean”] = (s[“mean”] * 100).round(1)
print(f”\n by {col}:\n{s.rename(columns={‘size’:’n’,’mean’:’acc%’}).to_string()}”)
print(“\n– judge decision breakdown –“)
print(res.how.value_counts().to_string())
errs = res[res.correct == 0]
if len(errs):
print(“\n– sample failures –“)
for _, r in errs.head(5).iterrows():
print(f” [{r.code}] gold={r.gold!r:>14} pred={r[‘pred’]!r:>20} ({r.how})”)
return t
BACKEND = make_backend()
RES = run_eval(RECORDS, BACKEND)
PER_CAP = report(RES, BACKEND.name)
LEADERBOARD = {
“GPT-5.6-Sol”: [59.7, 69.7, 62.4, 62.1, 55.5, 76.7, 67.0, 55.9, 60.0, 54.9, 26.9],
“Kimi K3”: [58.5, 68.2, 59.7, 59.4, 52.4, 70.3, 59.1, 55.9, 53.3, 61.2, 41.7],
“Claude-Fable-5”: [57.2, 58.5, 52.9, 60.9, 51.5, 70.4, 56.1, 51.6, 59.8, 64.3, 45.0],
“Gemini-3.1-Pro”: [56.2, 58.8, 56.9, 61.8, 50.0, 52.7, 61.7, 54.8, 61.2, 64.3, 40.6],
“Seed-2.1-Pro”: [55.0, 57.6, 51.2, 58.2, 43.6, 50.0, 59.5, 56.6, 60.4, 66.7, 49.8],
“Qwen3.5-397B-A17B”:[47.5, 55.2, 49.1, 53.0, 44.6, 46.7, 49.8, 44.8, 50.2, 52.9, 26.9],
“Gemma-4-31B”: [40.7, 42.7, 33.9, 40.3, 39.1, 44.9, 43.7, 39.0, 45.9, 46.7, 32.1],
“GLM-4.6V”: [32.5, 35.2, 31.8, 35.2, 29.1, 30.6, 34.8, 29.3, 33.7, 39.2, 26.9],
}
LB = pd.DataFrame(LEADERBOARD, index=[“Overall”] + CODE_ORDER).T
print(“\n” + “=” * 78)
print(“§10 OFFICIAL LEADERBOARD (subset, accuracy %)”)
print(“=” * 78)
print(LB.to_string())
print(“\nNote the structural finding from the card: Hallu is the weakest column ”
“almost everywhere,\nand models with near-identical Overall scores have ”
“very different capability profiles.”)
def radar(per_cap_df, label, compare=(“GPT-5.6-Sol”, “Gemma-4-31B”)):
codes = [c for c in CODE_ORDER if c in set(per_cap_df.code)]
if len(codes) < 3:
print(“[radar] need >=3 capabilities”); return
vals = per_cap_df.set_index(“code”).acc.reindex(codes).fillna(0).tolist()
ang = np.linspace(0, 2 * np.pi, len(codes), endpoint=False).tolist()
close = lambda v: v + v[:1]
fig, ax = plt.subplots(figsize=(6.4, 6.4), subplot_kw=dict(polar=True))
ax.plot(close(ang), close(vals), lw=2.4, color=”#C44E52″, label=label)
ax.fill(close(ang), close(vals), alpha=.18, color=”#C44E52″)
for m in compare:
if m in LB.index:
v = LB.loc[m, codes].tolist()
ax.plot(close(ang), close(v), lw=1.3, ls=”–“, alpha=.85, label=m)
ax.set_xticks(ang)
ax.set_xticklabels(codes)
ax.set_ylim(0, 100)
ax.set_yticks([20, 40, 60, 80])
ax.set_title(“PerceptionBench capability profile”, pad=24)
ax.legend(loc=”upper right”, bbox_to_anchor=(1.32, 1.12), fontsize=8)
plt.tight_layout(); plt.show()
if CFG[“SHOW_PLOTS”]:
radar(PER_CAP, BACKEND.name)
fig, ax = plt.subplots(figsize=(7, 3.2))
s = LB[“Overall”].sort_values()
ax.barh(s.index, s.values, color=”#8C8C8C”)
ax.barh([BACKEND.name], [RES.correct.mean() * 100], color=”#C44E52″)
ax.axvline(60, ls=”–“, c=”k”, lw=1)
ax.text(60.5, -.4, “60% ceiling: unbeaten”, fontsize=8)
ax.set_xlabel(“Overall accuracy (%)”); ax.set_title(“Your run vs. the leaderboard”)
plt.tight_layout(); plt.show()
tag = re.sub(r”[^A-Za-z0-9_.-]”, “_”, BACKEND.name)
p_pred = os.path.join(CFG[“OUT_DIR”], f”predictions_{tag}.jsonl”)
p_cap = os.path.join(CFG[“OUT_DIR”], f”per_capability_{tag}.csv”)
p_meta = os.path.join(CFG[“OUT_DIR”], f”run_meta_{tag}.json”)
with open(p_pred, “w”) as f:
for _, r in RES.iterrows():
f.write(json.dumps(r.to_dict(), default=str) + “\n”)
PER_CAP.to_csv(p_cap, index=False)
json.dump({“config”: {k: v for k, v in CFG.items() if “KEY” not in k},
“backend”: BACKEND.name, “n_questions”: len(RES),
“rows_scanned”: N_SCANNED,
“overall_acc”: float(RES.correct.mean() * 100),
“ci95”: list(bootstrap_ci(RES.correct)),
“timestamp”: time.strftime(“%Y-%m-%dT%H:%M:%S”)},
open(p_meta, “w”), indent=2)
print(f”\n[export] {p_pred}\n[export] {p_cap}\n[export] {p_meta}”)
print(“\n” + “=” * 78)
print(“DONE. Next steps:”)
print(” 1) CFG[‘BACKEND’]=’api’ + PB_API_KEY/PB_API_BASE/PB_API_MODEL -> score a real MLLM”)
print(” 2) CFG[‘BACKEND’]=’local’ on a GPU runtime -> score an open 2-3B VLM”)
print(” 3) CFG[‘JUDGE’]=’llm’ -> reproduce the paper’s LLM-as-judge protocol”)
print(” 4) Raise N_PER_CATEGORY / MAX_SCAN, or LOAD_MODE=’full’ for all 3,000 rows”)
print(” 5) Ablations worth running: crop-to-region vs. full image, image resolution”)
print(” sweep (MAX_IMAGE_SIDE 512/1024/2048), and CoT-on vs. CoT-off prompts”)
print(“=” * 78)
print(f”\n[eval] backend = {backend.name} on {len(records)} questions”)
t0 = time.time()
preds = backend.predict_batch(records)
rows = []
for rec, raw in zip(records, preds):
if CFG[“JUDGE”] == “llm” and CFG[“API_KEY”]:
ok, how = llm_judge(raw, rec[“answer”], rec[“problem”])
else:
ok, how = rule_judge(raw, rec[“answer”], CFG[“NUM_REL_TOL”])
rows.append(dict(index=rec[“index”], code=rec[“code”], category=rec[“category”],
source_bmk=rec[“source_bmk”], n_images=rec[“n_images”],
q_chars=rec[“q_chars”], max_side=rec[“max_side”],
ans_type=answer_type(rec[“answer”]),
gold=rec[“answer”], pred=extract_answer(raw),
raw=str(raw)[:2000], correct=ok, how=how))
print(f”[eval] done in {time.time()-t0:.1f}s”)
return pd.DataFrame(rows)
def bootstrap_ci(vals, n_boot=4000, seed=0):
a = np.asarray(vals, dtype=float)
if a.size == 0:
return (float(“nan”), float(“nan”))
rng = np.random.default_rng(seed)
means = a[rng.integers(0, a.size, (n_boot, a.size))].mean(axis=1)
return tuple(np.percentile(means, [2.5, 97.5]) * 100)
def report(res, label):
print(“\n” + “=” * 78)
print(f”§9 RESULTS — {label}”)
print(“=” * 78)
lo, hi = bootstrap_ci(res.correct)
print(f”\nOVERALL accuracy: {res.correct.mean()*100:5.1f}% ”
f”95% CI [{lo:.1f}, {hi:.1f}] (n={len(res)})”)
print(“(card: no frontier model exceeds 60% overall)\n”)
print(“– per atomic capability –“)
tab = []
for code, g in res.groupby(“code”):
l, h = bootstrap_ci(g.correct)
tab.append(dict(code=code, capability=CODE_FULL.get(code, code),
n=len(g), acc=g.correct.mean() * 100, lo=l, hi=h))
t = pd.DataFrame(tab).sort_values(“acc”, ascending=False)
print(t.to_string(index=False, float_format=lambda x: f”{x:6.1f}”))
print(“\n– difficulty slices –“)
res = res.copy()
res[“img_bucket”] = np.where(res.n_images > 1, “multi-image”, “single-image”)
res[“res_bucket”] = pd.cut(res.max_side, [0, 800, 1600, 10**6],
labels=[“<800px”, “800-1600px”, “>1600px”])
for col in [“img_bucket”, “res_bucket”, “ans_type”]:
s = res.groupby(col, observed=True).correct.agg([“size”, “mean”])
s[“mean”] = (s[“mean”] * 100).round(1)
print(f”\n by {col}:\n{s.rename(columns={‘size’:’n’,’mean’:’acc%’}).to_string()}”)
print(“\n– judge decision breakdown –“)
print(res.how.value_counts().to_string())
errs = res[res.correct == 0]
if len(errs):
print(“\n– sample failures –“)
for _, r in errs.head(5).iterrows():
print(f” [{r.code}] gold={r.gold!r:>14} pred={r[‘pred’]!r:>20} ({r.how})”)
return t
BACKEND = make_backend()
RES = run_eval(RECORDS, BACKEND)
PER_CAP = report(RES, BACKEND.name)
LEADERBOARD = {
“GPT-5.6-Sol”: [59.7, 69.7, 62.4, 62.1, 55.5, 76.7, 67.0, 55.9, 60.0, 54.9, 26.9],
“Kimi K3”: [58.5, 68.2, 59.7, 59.4, 52.4, 70.3, 59.1, 55.9, 53.3, 61.2, 41.7],
“Claude-Fable-5”: [57.2, 58.5, 52.9, 60.9, 51.5, 70.4, 56.1, 51.6, 59.8, 64.3, 45.0],
“Gemini-3.1-Pro”: [56.2, 58.8, 56.9, 61.8, 50.0, 52.7, 61.7, 54.8, 61.2, 64.3, 40.6],
“Seed-2.1-Pro”: [55.0, 57.6, 51.2, 58.2, 43.6, 50.0, 59.5, 56.6, 60.4, 66.7, 49.8],
“Qwen3.5-397B-A17B”:[47.5, 55.2, 49.1, 53.0, 44.6, 46.7, 49.8, 44.8, 50.2, 52.9, 26.9],
“Gemma-4-31B”: [40.7, 42.7, 33.9, 40.3, 39.1, 44.9, 43.7, 39.0, 45.9, 46.7, 32.1],
“GLM-4.6V”: [32.5, 35.2, 31.8, 35.2, 29.1, 30.6, 34.8, 29.3, 33.7, 39.2, 26.9],
}
LB = pd.DataFrame(LEADERBOARD, index=[“Overall”] + CODE_ORDER).T
print(“\n” + “=” * 78)
print(“§10 OFFICIAL LEADERBOARD (subset, accuracy %)”)
print(“=” * 78)
print(LB.to_string())
print(“\nNote the structural finding from the card: Hallu is the weakest column ”
“almost everywhere,\nand models with near-identical Overall scores have ”
“very different capability profiles.”)
def radar(per_cap_df, label, compare=(“GPT-5.6-Sol”, “Gemma-4-31B”)):
codes = [c for c in CODE_ORDER if c in set(per_cap_df.code)]
if len(codes) < 3:
print(“[radar] need >=3 capabilities”); return
vals = per_cap_df.set_index(“code”).acc.reindex(codes).fillna(0).tolist()
ang = np.linspace(0, 2 * np.pi, len(codes), endpoint=False).tolist()
close = lambda v: v + v[:1]
fig, ax = plt.subplots(figsize=(6.4, 6.4), subplot_kw=dict(polar=True))
ax.plot(close(ang), close(vals), lw=2.4, color=”#C44E52″, label=label)
ax.fill(close(ang), close(vals), alpha=.18, color=”#C44E52″)
for m in compare:
if m in LB.index:
v = LB.loc[m, codes].tolist()
ax.plot(close(ang), close(v), lw=1.3, ls=”–“, alpha=.85, label=m)
ax.set_xticks(ang)
ax.set_xticklabels(codes)
ax.set_ylim(0, 100)
ax.set_yticks([20, 40, 60, 80])
ax.set_title(“PerceptionBench capability profile”, pad=24)
ax.legend(loc=”upper right”, bbox_to_anchor=(1.32, 1.12), fontsize=8)
plt.tight_layout(); plt.show()
if CFG[“SHOW_PLOTS”]:
radar(PER_CAP, BACKEND.name)
fig, ax = plt.subplots(figsize=(7, 3.2))
s = LB[“Overall”].sort_values()
ax.barh(s.index, s.values, color=”#8C8C8C”)
ax.barh([BACKEND.name], [RES.correct.mean() * 100], color=”#C44E52″)
ax.axvline(60, ls=”–“, c=”k”, lw=1)
ax.text(60.5, -.4, “60% ceiling: unbeaten”, fontsize=8)
ax.set_xlabel(“Overall accuracy (%)”); ax.set_title(“Your run vs. the leaderboard”)
plt.tight_layout(); plt.show()
tag = re.sub(r”[^A-Za-z0-9_.-]”, “_”, BACKEND.name)
p_pred = os.path.join(CFG[“OUT_DIR”], f”predictions_{tag}.jsonl”)
p_cap = os.path.join(CFG[“OUT_DIR”], f”per_capability_{tag}.csv”)
p_meta = os.path.join(CFG[“OUT_DIR”], f”run_meta_{tag}.json”)
with open(p_pred, “w”) as f:
for _, r in RES.iterrows():
f.write(json.dumps(r.to_dict(), default=str) + “\n”)
PER_CAP.to_csv(p_cap, index=False)
json.dump({“config”: {k: v for k, v in CFG.items() if “KEY” not in k},
“backend”: BACKEND.name, “n_questions”: len(RES),
“rows_scanned”: N_SCANNED,
“overall_acc”: float(RES.correct.mean() * 100),
“ci95”: list(bootstrap_ci(RES.correct)),
“timestamp”: time.strftime(“%Y-%m-%dT%H:%M:%S”)},
open(p_meta, “w”), indent=2)
print(f”\n[export] {p_pred}\n[export] {p_cap}\n[export] {p_meta}”)
print(“\n” + “=” * 78)
print(“DONE. Next steps:”)
print(” 1) CFG[‘BACKEND’]=’api’ + PB_API_KEY/PB_API_BASE/PB_API_MODEL -> score a real MLLM”)
print(” 2) CFG[‘BACKEND’]=’local’ on a GPU runtime -> score an open 2-3B VLM”)
print(” 3) CFG[‘JUDGE’]=’llm’ -> reproduce the paper’s LLM-as-judge protocol”)
print(” 4) Raise N_PER_CATEGORY / MAX_SCAN, or LOAD_MODE=’full’ for all 3,000 rows”)
print(” 5) Ablations worth running: crop-to-region vs. full image, image resolution”)
print(” sweep (MAX_IMAGE_SIDE 512/1024/2048), and CoT-on vs. CoT-off prompts”)
print(“=” * 78)
