Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Per-case Detailed / Overview / Physical scores for the Text-to-Scene re-run (user, 2026-10-16). | |
| The internal leaderboard publishes one score per model and case. The public ranking also shows the three | |
| components of that score (0.2 x Detailed + 0.6 x Overview + 0.2 x Physical), so this reads them from the very | |
| result.json each case score comes from. It does not pick results itself: it imports the run-status page's own | |
| code and wraps its scoring step, which already decides which result counts (the latest score folder, and only | |
| when it scored the case's current valid attempt; a failed content check scores 0). Nothing is written. | |
| Run where the status page's code lives, with its folder as the argument; the JSON goes to stdout: | |
| ssh <host> python3 - <status-page-dir> < scripts/t2s_components.py > ../t2s-src/components.json | |
| """ | |
| import json | |
| import sys | |
| import time | |
| from pathlib import Path | |
| sys.path.insert(0, sys.argv[1]) | |
| import server # noqa: E402 (the status page; importing it does not start the server) | |
| scoring = server.sandbox_scoring | |
| def with_components(run, rows, score_dirs): | |
| out = scoring(run, rows, score_dirs) | |
| found = {} | |
| for sd in score_dirs: # later folders win, as in the status page | |
| for rj in (sd / "batches" / "text" / "results").glob("*-long/result.json") if sd.is_dir() else (): | |
| try: | |
| r = json.load(open(rj)) | |
| except Exception: | |
| continue | |
| comps = (r.get("primary_score") or {}).get("components") or {} | |
| found[rj.parent.name] = { | |
| "umap": Path(str(r.get("candidate_umap") or "")).name, | |
| "score": r.get("overall_score"), | |
| "detailed": (comps.get("semantic") or {}).get("score"), | |
| "overview": (comps.get("holistic_overview") or {}).get("score"), | |
| "physical": (comps.get("physics") or {}).get("score"), | |
| "weights": {k: (v or {}).get("weight") for k, v in comps.items()}, | |
| } | |
| for e, g in zip(out["cases"], rows): | |
| if not e["case"].endswith("-long") or e.get("status") != "done": | |
| continue | |
| r = found.get(e["case"]) | |
| cur = sorted((run / "cases" / e["case"] / f"attempt-{g.get('attempt')}" / "episode").glob("*.umap")) | |
| if r and cur and r["umap"] == cur[-1].name: # the result the status page counted | |
| e["components"] = {k: r[k] for k in ("detailed", "overview", "physical", "score", "weights")} | |
| return out | |
| server.sandbox_scoring = with_components | |
| snap = server.snapshot() | |
| print(json.dumps({ | |
| "generated_at": snap["generated_at"], | |
| "when": time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime(snap["generated_at"])), | |
| "models": [{"key": m["key"], "state": (m.get("scoring") or {}).get("state"), | |
| "cases": [c for c in (m.get("scoring") or {}).get("cases") or [] if c["case"].endswith("-long")]} | |
| for m in snap["models"]], | |
| })) | |