Spaces:
Running
Running
File size: 2,992 Bytes
988be21 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | #!/usr/bin/env python3
"""Per-case Detailed / Overview / Physical scores for the Text-to-Scene re-run (user, 2026-10-16).
The internal leaderboard publishes one score per model and case. The public ranking also shows the three
components of that score (0.2 x Detailed + 0.6 x Overview + 0.2 x Physical), so this reads them from the very
result.json each case score comes from. It does not pick results itself: it imports the run-status page's own
code and wraps its scoring step, which already decides which result counts (the latest score folder, and only
when it scored the case's current valid attempt; a failed content check scores 0). Nothing is written.
Run where the status page's code lives, with its folder as the argument; the JSON goes to stdout:
ssh <host> python3 - <status-page-dir> < scripts/t2s_components.py > ../t2s-src/components.json
"""
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, sys.argv[1])
import server # noqa: E402 (the status page; importing it does not start the server)
scoring = server.sandbox_scoring
def with_components(run, rows, score_dirs):
out = scoring(run, rows, score_dirs)
found = {}
for sd in score_dirs: # later folders win, as in the status page
for rj in (sd / "batches" / "text" / "results").glob("*-long/result.json") if sd.is_dir() else ():
try:
r = json.load(open(rj))
except Exception:
continue
comps = (r.get("primary_score") or {}).get("components") or {}
found[rj.parent.name] = {
"umap": Path(str(r.get("candidate_umap") or "")).name,
"score": r.get("overall_score"),
"detailed": (comps.get("semantic") or {}).get("score"),
"overview": (comps.get("holistic_overview") or {}).get("score"),
"physical": (comps.get("physics") or {}).get("score"),
"weights": {k: (v or {}).get("weight") for k, v in comps.items()},
}
for e, g in zip(out["cases"], rows):
if not e["case"].endswith("-long") or e.get("status") != "done":
continue
r = found.get(e["case"])
cur = sorted((run / "cases" / e["case"] / f"attempt-{g.get('attempt')}" / "episode").glob("*.umap"))
if r and cur and r["umap"] == cur[-1].name: # the result the status page counted
e["components"] = {k: r[k] for k in ("detailed", "overview", "physical", "score", "weights")}
return out
server.sandbox_scoring = with_components
snap = server.snapshot()
print(json.dumps({
"generated_at": snap["generated_at"],
"when": time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(snap["generated_at"])),
"models": [{"key": m["key"], "state": (m.get("scoring") or {}).get("state"),
"cases": [c for c in (m.get("scoring") or {}).get("cases") or [] if c["case"].endswith("-long")]}
for m in snap["models"]],
}))
|