File size: 16,722 Bytes
d19188c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
"""Task-level checks through an OpenAI-compatible endpoint (the way an agent harness talks to the model).

Usage: task_eval.py <base_url> <model_id> <out.json> [--suites gsm8k,humaneval,json,tools] [--n-gsm 50] [--n-he 40]
Greedy decoding (temperature 0) so that variants are compared on the same footing.
"""

import json
import os
import re
import subprocess
import sys
import tempfile
import time
import urllib.request

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from scenarios import TOOLS, run_tool  # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONTROL = re.compile(r"<\|[a-z_0-9]+\|>")


def chat(base, model, messages, tools=None, max_tokens=800, effort="low", temperature=0.0, extra=None):
    body = {"model": model, "messages": messages, "max_tokens": max_tokens, "temperature": temperature,
            "reasoning_effort": effort}
    if tools:
        body["tools"] = tools
    if extra:
        body.update(extra)
    req = urllib.request.Request(base.rstrip("/") + "/chat/completions", data=json.dumps(body).encode(),
                                 headers={"content-type": "application/json"})
    t0 = time.time()
    with urllib.request.urlopen(req, timeout=900) as r:
        data = json.load(r)
    choice = data["choices"][0]
    return choice["message"], choice.get("finish_reason"), data.get("usage", {}), time.time() - t0


def leaked(msg):
    return bool(CONTROL.search((msg.get("content") or "")))


# ------------------------------------------------------------------ GSM8K
def number(text):
    m = re.findall(r"Answer:\s*\$?\s*(-?[\d,]*\.?\d+)", text or "")
    if not m:
        m = re.findall(r"(-?[\d,]*\.?\d+)", text or "")
    if not m:
        return None
    try:
        return float(m[-1].replace(",", ""))
    except ValueError:
        return None


def suite_gsm8k(base, model, n):
    import pyarrow.parquet as pq
    rows = pq.read_table(os.path.join(ROOT, "data/evals/gsm8k_test.parquet")).to_pylist()
    step = len(rows) // n
    out = []
    for i in range(n):
        row = rows[i * step]
        gold = float(row["answer"].split("####")[-1].strip().replace(",", ""))
        msg, fin, usage, dt = chat(base, model, [{"role": "user", "content": row["question"] +
                                   "\n\nSolve it, then give the final answer on the last line as 'Answer: <number>'."}],
                                   max_tokens=900, effort="low")
        pred = number(msg.get("content"))
        out.append(dict(i=i * step, ok=pred is not None and abs(pred - gold) < 1e-4, pred=pred, gold=gold, finish=fin,
                        leaked=leaked(msg), tokens=usage.get("completion_tokens"), seconds=round(dt, 1)))
    return out


# ------------------------------------------------------------------ HumanEval
SANDBOX = """(version 1)
(allow default)
(deny network*)
(deny file-write*)
(allow file-write* (subpath "{tmp}") (literal "/dev/null") (literal "/dev/dtracehelper"))
"""


def run_python(code, timeout=15):
    with tempfile.TemporaryDirectory() as tmp:
        tmp = os.path.realpath(tmp)
        path = os.path.join(tmp, "prog.py")
        open(path, "w").write(code)
        try:
            p = subprocess.run(["/usr/bin/sandbox-exec", "-p", SANDBOX.format(tmp=tmp), sys.executable, "-I", "-B", path],
                               capture_output=True, text=True, timeout=timeout, cwd=tmp)
            return p.returncode == 0, (p.stderr or "")[-300:]
        except subprocess.TimeoutExpired:
            return False, "timeout"


def extract_code(text):
    blocks = re.findall(r"```(?:python|py)?\n(.*?)```", text or "", flags=re.S)
    return max(blocks, key=len) if blocks else (text or "")


def suite_humaneval(base, model, n):
    import pyarrow.parquet as pq
    rows = pq.read_table(os.path.join(ROOT, "data/evals/humaneval.parquet")).to_pylist()
    step = len(rows) // n
    out = []
    for i in range(n):
        row = rows[i * step]
        msg, fin, usage, dt = chat(base, model, [{"role": "user", "content":
                                   "Complete this Python function. Reply with the full function (including the signature "
                                   "and any imports) in one ```python block and nothing else.\n\n" + row["prompt"]}],
                                   max_tokens=900, effort="low")
        code = extract_code(msg.get("content"))
        ok, err = run_python(code + "\n\n" + row["test"] + f"\n\ncheck({row['entry_point']})\n")
        out.append(dict(task=row["task_id"], ok=ok, err=None if ok else err, finish=fin, leaked=leaked(msg),
                        tokens=usage.get("completion_tokens"), seconds=round(dt, 1)))
    return out


# ------------------------------------------------------------------ structured output
JSON_CASES = [
    ("Return a JSON object with keys \"city\" (string) and \"population_millions\" (number) for the capital of Japan. Output only JSON.",
     lambda d: isinstance(d, dict) and d.get("city") == "Tokyo" and isinstance(d.get("population_millions"), (int, float))),
    ("Output only a JSON array of the first six prime numbers.", lambda d: d == [2, 3, 5, 7, 11, 13]),
    ("Extract as JSON {\"name\": string, \"email\": string, \"age\": integer} from: 'Contact Maria Okafor (34) at maria.okafor@example.org.' Output only JSON.",
     lambda d: isinstance(d, dict) and d.get("name") == "Maria Okafor" and d.get("email") == "maria.okafor@example.org" and d.get("age") == 34),
    ("Output only JSON: an object mapping each of the words \"apple\", \"kiwi\", \"banana\" to its number of letters.",
     lambda d: d == {"apple": 5, "kiwi": 4, "banana": 6}),
    ("Convert to JSON with keys \"items\" (array of {\"sku\": string, \"qty\": integer}) and \"rush\" (boolean): 'Two of SKU A-17 and five of SKU B-02, rush delivery please.' Output only JSON.",
     lambda d: isinstance(d, dict) and d.get("rush") is True and sorted((x.get("sku"), x.get("qty")) for x in d.get("items", [])) == [("A-17", 2), ("B-02", 5)]),
    ("Output only JSON: {\"sorted\": [...]} with the numbers 42, 7, 19, 3 in ascending order.", lambda d: d == {"sorted": [3, 7, 19, 42]}),
    ("Return only a JSON object {\"sentiment\": \"positive\"|\"negative\"|\"neutral\", \"confidence\": number between 0 and 1} for: 'The update broke everything and support never answered.'",
     lambda d: isinstance(d, dict) and d.get("sentiment") == "negative" and isinstance(d.get("confidence"), (int, float)) and 0 <= d["confidence"] <= 1),
    ("Output only JSON: {\"date\": \"YYYY-MM-DD\"} for the 4th of July 2031.", lambda d: d == {"date": "2031-07-04"}),
    ("Output only a JSON object with a key \"quote\" whose value is the exact text: She said \"stop\" and left.\\ Then nothing.",
     lambda d: isinstance(d, dict) and d.get("quote") == 'She said "stop" and left.\\ Then nothing.'),
    ("Output only JSON: {\"tree\": {\"value\": 1, \"children\": [{\"value\": 2, \"children\": []}, {\"value\": 3, \"children\": [{\"value\": 4, \"children\": []}]}]}} but with every value doubled.",
     lambda d: d == {"tree": {"value": 2, "children": [{"value": 4, "children": []}, {"value": 6, "children": [{"value": 8, "children": []}]}]}}),
]


def parse_json(text):
    text = (text or "").strip()
    m = re.search(r"```(?:json)?\n(.*?)```", text, flags=re.S)
    if m:
        text = m.group(1).strip()
    return json.loads(text)


def suite_json(base, model, _n=None):
    out = []
    for i, (prompt, check) in enumerate(JSON_CASES):
        msg, fin, usage, dt = chat(base, model, [{"role": "user", "content": prompt}], max_tokens=1200, effort="none")
        try:
            data = parse_json(msg.get("content")); valid = True
            ok = bool(check(data))
        except Exception:  # noqa: BLE001
            valid, ok = False, False
        out.append(dict(i=i, ok=ok, valid_json=valid, finish=fin, leaked=leaked(msg), seconds=round(dt, 1),
                        content=None if ok else (msg.get("content") or "")[:200]))
    return out


# ------------------------------------------------------------------ tool use
def args_of(call):
    a = call["function"].get("arguments")
    return json.loads(a) if isinstance(a, str) else a


SINGLE = [  # (tools, user, expected tool, argument check)
    (["get_weather", "calculator"], "What's the weather in Tokyo in fahrenheit?", "get_weather",
     lambda a: a.get("city", "").lower() == "tokyo" and a.get("unit") == "fahrenheit"),
    (["get_weather", "web_search"], "Look up the weather for Cairo.", "get_weather", lambda a: a.get("city", "").lower() == "cairo"),
    (["calculator", "web_search"], "Use the calculator to compute 48 * 365.", "calculator",
     lambda a: re.sub(r"\s", "", a.get("expression", "")) in ("48*365", "365*48")),
    (["terminal", "read_file", "write_file"], "List the files in the current directory.", "terminal", lambda a: "ls" in a.get("command", "")),
    (["terminal", "read_file", "write_file"], "Show me the contents of app/config.py.", None,
     lambda a: "app/config.py" in json.dumps(a)),
    (["terminal", "read_file", "write_file"], "Create a file notes/todo.txt containing exactly the text: buy milk", None,
     lambda a: (a.get("path", "").endswith("notes/todo.txt") and a.get("content", "").strip() == "buy milk")
     or ("todo.txt" in a.get("command", "") and "buy milk" in a.get("command", ""))),
    (["search_orders"], "Find customer C-88's shipped orders over 100 dollars that are tagged gift.", "search_orders",
     lambda a: a.get("customer_id") == "C-88" and a.get("status") == "shipped" and (a.get("filters") or {}).get("min_total") == 100
     and (a.get("filters") or {}).get("tags") == ["gift"]),
    (["search_orders", "calculator"], "List open orders for customer C-19.", "search_orders",
     lambda a: a.get("customer_id") == "C-19" and a.get("status") == "open"),
    (["web_search", "get_weather"], "Search the web for the MLX array framework, three results at most.", "web_search",
     lambda a: "mlx" in a.get("query", "").lower() and a.get("max_results", 3) <= 3),
    (["terminal"], "Run the test suite with pytest in quiet mode.", "terminal", lambda a: "pytest" in a.get("command", "")),
    (["write_file", "read_file"], "Write a file hello.py that prints the string He said \"hi\" on one line and a backslash \\ on the next.",
     "write_file", lambda a: a.get("path", "").endswith("hello.py") and "print" in a.get("content", "")),
    (["get_weather", "calculator", "web_search", "terminal", "read_file", "write_file", "search_orders"],
     "Use the weather tool for Oslo in celsius.", "get_weather", lambda a: a.get("city", "").lower() == "oslo" and a.get("unit", "celsius") == "celsius"),
]
NO_TOOL = [
    (["get_weather", "web_search"], "No lookup needed: what is 2 + 2?"),
    (["terminal", "read_file"], "Just say hello in French, no tools."),
]
MULTI = [  # (tools, system, user, check(final_text, calls))
    (["get_weather"], None, "Is it warmer in Lisbon or in Oslo right now? Use the tool.",
     lambda text, calls: "lisbon" in text.lower() and len(calls) >= 2),
    (["terminal", "read_file", "write_file"], "You are a coding agent in a sandboxed repository. Use the tools; keep answers short.",
     "What is the value of RETRIES in app/config.py?", lambda text, calls: "3" in text and len(calls) >= 1),
    (["terminal", "read_file", "write_file"], "You are a coding agent in a sandboxed repository. Use the tools; keep answers short.",
     "Open docs/ARCHITECTURE.md and summarise it. If it does not exist, say so.",
     lambda text, calls: any(w in text.lower() for w in ("not exist", "no such", "doesn't exist", "does not exist", "isn't", "missing", "no `docs", "not found", "there is no"))),
    (["calculator"], None, "Use the calculator: what is (1234 * 57) - 999, and what is that divided by 7?",
     lambda text, calls: "69339" in text.replace(",", "") and len(calls) >= 1),
    (["search_orders"], None, "How many orders does customer C-88 have in total, and which is the most expensive?",
     lambda text, calls: "A-1042" in text and len(calls) >= 1),
]


def suite_tools(base, model, _n=None):
    import scenarios
    out = []
    for i, (names, user, want, check) in enumerate(SINGLE):
        msg, fin, usage, dt = chat(base, model, [{"role": "user", "content": user}], tools=[TOOLS[t] for t in names],
                                   max_tokens=500, effort="low")
        calls = msg.get("tool_calls") or []
        rec = dict(kind="single", i=i, n_calls=len(calls), finish=fin, leaked=leaked(msg), seconds=round(dt, 1))
        try:
            a = args_of(calls[0]) if calls else None
            rec["valid_json"] = a is not None
            rec["right_tool"] = bool(calls) and (want is None or calls[0]["function"]["name"] == want)
            rec["ok"] = bool(rec["right_tool"] and check(a) and fin == "tool_calls")
        except Exception as exc:  # noqa: BLE001
            rec.update(valid_json=False, right_tool=False, ok=False, error=str(exc)[:120])
        if not rec["ok"]:
            rec["got"] = json.dumps(calls)[:300] if calls else (msg.get("content") or "")[:200]
        out.append(rec)
    for i, (names, user) in enumerate(NO_TOOL):
        msg, fin, usage, dt = chat(base, model, [{"role": "user", "content": user}], tools=[TOOLS[t] for t in names],
                                   max_tokens=300, effort="low")
        calls = msg.get("tool_calls") or []
        out.append(dict(kind="no_tool", i=i, ok=not calls and bool((msg.get("content") or "").strip()) and fin == "stop",
                        n_calls=len(calls), finish=fin, leaked=leaked(msg), seconds=round(dt, 1)))
    for i, (names, system, user, check) in enumerate(MULTI):
        scenarios.FILES.update(ORIG_FILES)
        for k in list(scenarios.FILES):
            if k not in ORIG_FILES:
                del scenarios.FILES[k]
        messages = ([{"role": "system", "content": system}] if system else []) + [{"role": "user", "content": user}]
        all_calls, rounds, leak, fin, t0, err = [], 0, False, None, time.time(), None
        try:
            for rounds in range(1, 13):
                msg, fin, usage, dt = chat(base, model, messages, tools=[TOOLS[t] for t in names], max_tokens=900, effort="low")
                leak = leak or leaked(msg)
                calls = msg.get("tool_calls") or []
                clean = {"role": "assistant", "content": msg.get("content") or ""}
                if calls:
                    clean["tool_calls"] = calls
                messages.append(clean)
                if not calls:
                    break
                all_calls += calls
                for c in calls:
                    messages.append({"role": "tool", "tool_call_id": c.get("id"), "name": c["function"]["name"],
                                     "content": run_tool(c["function"]["name"], args_of(c) or {})})
            final = messages[-1].get("content") or ""
            ok = bool(messages[-1]["role"] == "assistant" and check(final, all_calls))
        except Exception as exc:  # noqa: BLE001
            ok, final, err = False, "", str(exc)[:200]
        out.append(dict(kind="multi", i=i, ok=ok, rounds=rounds, n_calls=len(all_calls), finish=fin, leaked=leak,
                        seconds=round(time.time() - t0, 1), final=final[:240], error=err))
    return out


import scenarios as _sc  # noqa: E402

ORIG_FILES = dict(_sc.FILES)
SUITES = {"gsm8k": suite_gsm8k, "humaneval": suite_humaneval, "json": suite_json, "tools": suite_tools}


def main():
    args = sys.argv[1:]
    base, model, out_path = args[:3]
    suites, n = ["gsm8k", "humaneval", "json", "tools"], {"gsm8k": 50, "humaneval": 40}
    rest = args[3:]
    while rest:
        if rest[0] == "--suites":
            suites = rest[1].split(","); rest = rest[2:]
        elif rest[0] == "--n-gsm":
            n["gsm8k"] = int(rest[1]); rest = rest[2:]
        elif rest[0] == "--n-he":
            n["humaneval"] = int(rest[1]); rest = rest[2:]
        else:
            raise SystemExit("unknown flag " + rest[0])
    results = json.load(open(out_path)) if os.path.exists(out_path) else {}
    for s in suites:
        if s in results:
            continue
        t0 = time.time()
        rows = SUITES[s](base, model, n.get(s))
        results[s] = dict(passed=sum(r["ok"] for r in rows), total=len(rows), leaked=sum(bool(r.get("leaked")) for r in rows),
                          seconds=round(time.time() - t0), rows=rows)
        json.dump(results, open(out_path, "w"), indent=1)
        print(f"{s:10s} {results[s]['passed']}/{results[s]['total']}  control-token leaks {results[s]['leaked']}  {results[s]['seconds']}s", flush=True)


if __name__ == "__main__":
    main()