[{"id":"coldfusion_4bit","label":"Qwen3.8 27B Cold Fusion 4-bit","original":{"attempts":8,"accepted_passes":7,"strict_passes":7,"delivered":7,"incomplete":1,"native_recorded_tokens":19759,"partial_observed_tokens":0,"native_phase_s":744.2459804168902,"partial_generation_s":0,"weighted_native_decode_tps":26.8247432172367,"known_wall_s":792.6303041648353,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":7,"strict_passes":7,"delivered":7,"incomplete":1,"native_recorded_tokens":19759,"partial_observed_tokens":0,"native_phase_s":744.2459804168902,"partial_generation_s":0,"weighted_native_decode_tps":26.8247432172367,"known_wall_s":792.6303041648353,"missing_wall_count":0},"original_tasks":[{"model":"coldfusion_4bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2834,"phase_s":104.83854616701137,"decode_s":103.15166574914474,"prefill_s":1.683212541975081,"decode_tps":27.474107949861175,"wall_s":111.42984987492673,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":27.474107949861175,"correct_answer_tps":27.474107949861175,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"coldfusion_4bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2830,"phase_s":137.94753412506543,"decode_s":137.02181058307178,"prefill_s":0.9214509999146685,"decode_tps":20.65364621849209,"wall_s":144.69247195799835,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":20.65364621849209,"correct_answer_tps":20.65364621849209,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"coldfusion_4bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2421,"phase_s":127.70372308290098,"decode_s":126.9838642498944,"prefill_s":0.7166417089756578,"decode_tps":19.06541444695414,"wall_s":132.58469766599592,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":19.06541444695414,"correct_answer_tps":19.06541444695414,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n intervals_by_user = {}\n\n for record in records:\n user = record[\"user\"]\n start = record[\"start\"]\n end = record[\"end\"]\n\n if end < start:\n raise ValueError(\"end must be greater than or equal to start\")\n\n intervals_by_user.setdefault(user, []).append((start, end))\n\n result = {}\n for user in sorted(intervals_by_user):\n intervals = intervals_by_user[user]\n intervals.sort(key=lambda interval: interval[0])\n\n total = 0\n if intervals:\n current_start, current_end = intervals[0]\n for start, end in intervals[1:]:\n if start > current_end:\n total += current_end - current_start\n current_start, current_end = start, end\n elif end > current_end:\n current_end = end\n total += current_end - current_start\n\n result[user] = total\n\n return result\n```"},{"model":"coldfusion_4bit","task":"batches","phase":"original","budget":4096,"passed":false,"score":0.0,"content_passed":false,"tokens":4096,"phase_s":155.8022630829364,"decode_s":155.29676941689104,"prefill_s":0.5024620840558782,"decode_tps":26.37530719653524,"wall_s":162.76648716605268,"wall_kind":"fresh command","finish":"cap","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"coldfusion_4bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1821,"phase_s":58.17921970796306,"decode_s":57.5474216240691,"prefill_s":0.6285909989383072,"decode_tps":31.643468093075608,"wall_s":63.30889937502798,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.643468093075608,"correct_answer_tps":31.643468093075608,"outcome":"pass","final_answer":"```sql\nWITH paid_order_metrics AS (\n SELECT\n o.id AS order_id,\n o.customer_id,\n COALESCE(\n (\n SELECT SUM(i.qty * i.unit_cents)\n FROM items i\n WHERE i.order_id = o.id\n ),\n 0\n ) AS gross_cents,\n COALESCE(\n (\n SELECT SUM(r.amount_cents)\n FROM refunds r\n WHERE r.order_id = o.id\n AND r.status = 'settled'\n ),\n 0\n ) AS refund_cents\n FROM orders o\n WHERE o.status = 'paid'\n)\nSELECT\n c.id AS customer_id,\n COUNT(pom.order_id) AS paid_order_count,\n COALESCE(SUM(pom.gross_cents - pom.refund_cents), 0) AS net_cents\nFROM customers c\nLEFT JOIN paid_order_metrics pom\n ON pom.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id;\n```"},{"model":"coldfusion_4bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2767,"phase_s":72.9316536670085,"decode_s":71.66218320897315,"prefill_s":1.26551537506748,"decode_tps":38.611717869816886,"wall_s":78.37881441693753,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":38.611717869816886,"correct_answer_tps":38.611717869816886,"outcome":"pass","final_answer":"{\"read_results\":[10,null,30,null,31,null,null,70],\"final_lru_to_mru\":[\"F\",\"H\",\"G\"],\"hits\":4,\"misses\":4}"},{"model":"coldfusion_4bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2016,"phase_s":60.665408417000435,"decode_s":59.65793345798738,"prefill_s":1.0043448329670355,"decode_tps":33.792655614189485,"wall_s":66.3072169169318,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":33.792655614189485,"correct_answer_tps":33.792655614189485,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\n \"L3\",\n \"L5\"\n ]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\n \"L7\"\n ]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [\n \"L6\"\n ]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\n \"L8\"\n ]\n }\n]"},{"model":"coldfusion_4bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":974,"phase_s":26.17763216700405,"decode_s":25.27440558397211,"prefill_s":0.9003089180914685,"decode_tps":38.537009179660664,"wall_s":33.16186679096427,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":38.537009179660664,"correct_answer_tps":38.537009179660664,"outcome":"pass","final_answer":"{\n \"A\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"B\": {\n \"eligible\": true,\n \"credit\": 8.5\n },\n \"C\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"D\": {\n \"eligible\": null,\n \"credit\": null\n },\n \"E\": {\n \"eligible\": true,\n \"credit\": null\n },\n \"F\": {\n \"eligible\": false,\n \"credit\": 0\n }\n}"}],"selected_latest_tasks":[{"model":"coldfusion_4bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2834,"phase_s":104.83854616701137,"decode_s":103.15166574914474,"prefill_s":1.683212541975081,"decode_tps":27.474107949861175,"wall_s":111.42984987492673,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":27.474107949861175,"correct_answer_tps":27.474107949861175,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"coldfusion_4bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2830,"phase_s":137.94753412506543,"decode_s":137.02181058307178,"prefill_s":0.9214509999146685,"decode_tps":20.65364621849209,"wall_s":144.69247195799835,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":20.65364621849209,"correct_answer_tps":20.65364621849209,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"coldfusion_4bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2421,"phase_s":127.70372308290098,"decode_s":126.9838642498944,"prefill_s":0.7166417089756578,"decode_tps":19.06541444695414,"wall_s":132.58469766599592,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":19.06541444695414,"correct_answer_tps":19.06541444695414,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n intervals_by_user = {}\n\n for record in records:\n user = record[\"user\"]\n start = record[\"start\"]\n end = record[\"end\"]\n\n if end < start:\n raise ValueError(\"end must be greater than or equal to start\")\n\n intervals_by_user.setdefault(user, []).append((start, end))\n\n result = {}\n for user in sorted(intervals_by_user):\n intervals = intervals_by_user[user]\n intervals.sort(key=lambda interval: interval[0])\n\n total = 0\n if intervals:\n current_start, current_end = intervals[0]\n for start, end in intervals[1:]:\n if start > current_end:\n total += current_end - current_start\n current_start, current_end = start, end\n elif end > current_end:\n current_end = end\n total += current_end - current_start\n\n result[user] = total\n\n return result\n```"},{"model":"coldfusion_4bit","task":"batches","phase":"original","budget":4096,"passed":false,"score":0.0,"content_passed":false,"tokens":4096,"phase_s":155.8022630829364,"decode_s":155.29676941689104,"prefill_s":0.5024620840558782,"decode_tps":26.37530719653524,"wall_s":162.76648716605268,"wall_kind":"fresh command","finish":"cap","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"coldfusion_4bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1821,"phase_s":58.17921970796306,"decode_s":57.5474216240691,"prefill_s":0.6285909989383072,"decode_tps":31.643468093075608,"wall_s":63.30889937502798,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.643468093075608,"correct_answer_tps":31.643468093075608,"outcome":"pass","final_answer":"```sql\nWITH paid_order_metrics AS (\n SELECT\n o.id AS order_id,\n o.customer_id,\n COALESCE(\n (\n SELECT SUM(i.qty * i.unit_cents)\n FROM items i\n WHERE i.order_id = o.id\n ),\n 0\n ) AS gross_cents,\n COALESCE(\n (\n SELECT SUM(r.amount_cents)\n FROM refunds r\n WHERE r.order_id = o.id\n AND r.status = 'settled'\n ),\n 0\n ) AS refund_cents\n FROM orders o\n WHERE o.status = 'paid'\n)\nSELECT\n c.id AS customer_id,\n COUNT(pom.order_id) AS paid_order_count,\n COALESCE(SUM(pom.gross_cents - pom.refund_cents), 0) AS net_cents\nFROM customers c\nLEFT JOIN paid_order_metrics pom\n ON pom.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id;\n```"},{"model":"coldfusion_4bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2767,"phase_s":72.9316536670085,"decode_s":71.66218320897315,"prefill_s":1.26551537506748,"decode_tps":38.611717869816886,"wall_s":78.37881441693753,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":38.611717869816886,"correct_answer_tps":38.611717869816886,"outcome":"pass","final_answer":"{\"read_results\":[10,null,30,null,31,null,null,70],\"final_lru_to_mru\":[\"F\",\"H\",\"G\"],\"hits\":4,\"misses\":4}"},{"model":"coldfusion_4bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2016,"phase_s":60.665408417000435,"decode_s":59.65793345798738,"prefill_s":1.0043448329670355,"decode_tps":33.792655614189485,"wall_s":66.3072169169318,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":33.792655614189485,"correct_answer_tps":33.792655614189485,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\n \"L3\",\n \"L5\"\n ]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\n \"L7\"\n ]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [\n \"L6\"\n ]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\n \"L8\"\n ]\n }\n]"},{"model":"coldfusion_4bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":974,"phase_s":26.17763216700405,"decode_s":25.27440558397211,"prefill_s":0.9003089180914685,"decode_tps":38.537009179660664,"wall_s":33.16186679096427,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":38.537009179660664,"correct_answer_tps":38.537009179660664,"outcome":"pass","final_answer":"{\n \"A\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"B\": {\n \"eligible\": true,\n \"credit\": 8.5\n },\n \"C\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"D\": {\n \"eligible\": null,\n \"credit\": null\n },\n \"E\": {\n \"eligible\": true,\n \"credit\": null\n },\n \"F\": {\n \"eligible\": false,\n \"credit\": 0\n }\n}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"youssofal_4bit","label":"Qwen3.8 27B Youssofal 4-bit","original":{"attempts":8,"accepted_passes":8,"strict_passes":8,"delivered":8,"incomplete":0,"native_recorded_tokens":15864,"partial_observed_tokens":0,"native_phase_s":513.2106527509168,"partial_generation_s":0,"weighted_native_decode_tps":31.389940335089513,"known_wall_s":575.8192865010351,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":8,"strict_passes":8,"delivered":8,"incomplete":0,"native_recorded_tokens":15864,"partial_observed_tokens":0,"native_phase_s":513.2106527509168,"partial_generation_s":0,"weighted_native_decode_tps":31.389940335089513,"known_wall_s":575.8192865010351,"missing_wall_count":0},"original_tasks":[{"model":"youssofal_4bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2256,"phase_s":79.63875637506135,"decode_s":77.80343100009486,"prefill_s":1.8309000009903684,"decode_tps":28.99615056818316,"wall_s":87.39822237496264,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.99615056818316,"correct_answer_tps":28.99615056818316,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"},{"model":"youssofal_4bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1837,"phase_s":74.75446829199791,"decode_s":73.79445979208685,"prefill_s":0.9553092499263585,"decode_tps":24.893467682745822,"wall_s":81.3416706660064,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":24.893467682745822,"correct_answer_tps":24.893467682745822,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"youssofal_4bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1897,"phase_s":61.52988020889461,"decode_s":60.98813279194292,"prefill_s":0.5366677079582587,"decode_tps":31.10441184470253,"wall_s":78.40983679203782,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.10441184470253,"correct_answer_tps":31.10441184470253,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for record in records:\n user = record['user']\n start = record['start']\n end = record['end']\n if end < start:\n raise ValueError(\"end < start\")\n if user not in user_intervals:\n user_intervals[user] = []\n user_intervals[user].append((start, end))\n\n result = {}\n for user in sorted(user_intervals):\n intervals = user_intervals[user]\n intervals.sort()\n total = 0\n if intervals:\n cur_start, cur_end = intervals[0]\n for s, e in intervals[1:]:\n if s <= cur_end:\n if e > cur_end:\n cur_end = e\n else:\n total += cur_end - cur_start\n cur_start, cur_end = s, e\n total += cur_end - cur_start\n result[user] = total\n return result\n```"},{"model":"youssofal_4bit","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1708,"phase_s":54.12057695793919,"decode_s":53.59467991592828,"prefill_s":0.5214275838807225,"decode_tps":31.868834792544106,"wall_s":60.29710462503135,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.868834792544106,"correct_answer_tps":31.868834792544106,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n if not graph:\n return []\n\n all_nodes = set(graph.keys())\n for prereqs in graph.values():\n for p in prereqs:\n all_nodes.add(p)\n\n prereqs = {}\n for node in all_nodes:\n if node in graph:\n prereqs[node] = set(graph[node])\n else:\n prereqs[node] = set()\n\n placed = set()\n batches = []\n\n while len(placed) < len(all_nodes):\n available = sorted(\n n for n in all_nodes\n if n not in placed and all(p in placed for p in prereqs[n])\n )\n if not available:\n raise ValueError(\"Cycle detected\")\n batches.append(available)\n placed.update(available)\n\n return batches\n```"},{"model":"youssofal_4bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1447,"phase_s":45.27044633298647,"decode_s":44.61111112497747,"prefill_s":0.6557280409615487,"decode_tps":32.4358654942767,"wall_s":51.28336904197931,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":32.4358654942767,"correct_answer_tps":32.4358654942767,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT id, customer_id\n FROM orders\n WHERE status = 'paid'\n),\norder_gross AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.id\n GROUP BY po.id\n),\norder_refunds AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(r.amount_cents), 0) AS total_refunded\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.id AND r.status = 'settled'\n GROUP BY po.id\n)\nSELECT\n c.id AS customer_id,\n COUNT(po.id) AS paid_order_count,\n COALESCE(SUM(og.gross - orf.total_refunded), 0) AS net_cents\nFROM customers c\nLEFT JOIN paid_orders po ON po.customer_id = c.id\nLEFT JOIN order_gross og ON og.order_id = po.id\nLEFT JOIN order_refunds orf ON orf.order_id = po.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"youssofal_4bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3009,"phase_s":84.6015653340146,"decode_s":83.2762271668762,"prefill_s":1.3216383749386296,"decode_tps":36.13276084146202,"wall_s":90.41939929197542,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":36.13276084146202,"correct_answer_tps":36.13276084146202,"outcome":"pass","final_answer":"{\"read_results\": [10, null, 30, null, 31, null, null, 70], \"final_lru_to_mru\": [\"F\", \"H\", \"G\"], \"hits\": 4, \"misses\": 4}"},{"model":"youssofal_4bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2579,"phase_s":81.54990187496878,"decode_s":80.51017791591585,"prefill_s":1.0358077499549836,"decode_tps":32.03321700137697,"wall_s":87.47647341701668,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":32.03321700137697,"correct_answer_tps":32.03321700137697,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"youssofal_4bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1131,"phase_s":31.745057375053875,"decode_s":30.806621124967933,"prefill_s":0.9350159580353647,"decode_tps":36.712886993093676,"wall_s":39.19321029202547,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":36.712886993093676,"correct_answer_tps":36.712886993093676,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"}],"selected_latest_tasks":[{"model":"youssofal_4bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2256,"phase_s":79.63875637506135,"decode_s":77.80343100009486,"prefill_s":1.8309000009903684,"decode_tps":28.99615056818316,"wall_s":87.39822237496264,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.99615056818316,"correct_answer_tps":28.99615056818316,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"},{"model":"youssofal_4bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1837,"phase_s":74.75446829199791,"decode_s":73.79445979208685,"prefill_s":0.9553092499263585,"decode_tps":24.893467682745822,"wall_s":81.3416706660064,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":24.893467682745822,"correct_answer_tps":24.893467682745822,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"youssofal_4bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1897,"phase_s":61.52988020889461,"decode_s":60.98813279194292,"prefill_s":0.5366677079582587,"decode_tps":31.10441184470253,"wall_s":78.40983679203782,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.10441184470253,"correct_answer_tps":31.10441184470253,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for record in records:\n user = record['user']\n start = record['start']\n end = record['end']\n if end < start:\n raise ValueError(\"end < start\")\n if user not in user_intervals:\n user_intervals[user] = []\n user_intervals[user].append((start, end))\n\n result = {}\n for user in sorted(user_intervals):\n intervals = user_intervals[user]\n intervals.sort()\n total = 0\n if intervals:\n cur_start, cur_end = intervals[0]\n for s, e in intervals[1:]:\n if s <= cur_end:\n if e > cur_end:\n cur_end = e\n else:\n total += cur_end - cur_start\n cur_start, cur_end = s, e\n total += cur_end - cur_start\n result[user] = total\n return result\n```"},{"model":"youssofal_4bit","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1708,"phase_s":54.12057695793919,"decode_s":53.59467991592828,"prefill_s":0.5214275838807225,"decode_tps":31.868834792544106,"wall_s":60.29710462503135,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.868834792544106,"correct_answer_tps":31.868834792544106,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n if not graph:\n return []\n\n all_nodes = set(graph.keys())\n for prereqs in graph.values():\n for p in prereqs:\n all_nodes.add(p)\n\n prereqs = {}\n for node in all_nodes:\n if node in graph:\n prereqs[node] = set(graph[node])\n else:\n prereqs[node] = set()\n\n placed = set()\n batches = []\n\n while len(placed) < len(all_nodes):\n available = sorted(\n n for n in all_nodes\n if n not in placed and all(p in placed for p in prereqs[n])\n )\n if not available:\n raise ValueError(\"Cycle detected\")\n batches.append(available)\n placed.update(available)\n\n return batches\n```"},{"model":"youssofal_4bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1447,"phase_s":45.27044633298647,"decode_s":44.61111112497747,"prefill_s":0.6557280409615487,"decode_tps":32.4358654942767,"wall_s":51.28336904197931,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":32.4358654942767,"correct_answer_tps":32.4358654942767,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT id, customer_id\n FROM orders\n WHERE status = 'paid'\n),\norder_gross AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.id\n GROUP BY po.id\n),\norder_refunds AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(r.amount_cents), 0) AS total_refunded\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.id AND r.status = 'settled'\n GROUP BY po.id\n)\nSELECT\n c.id AS customer_id,\n COUNT(po.id) AS paid_order_count,\n COALESCE(SUM(og.gross - orf.total_refunded), 0) AS net_cents\nFROM customers c\nLEFT JOIN paid_orders po ON po.customer_id = c.id\nLEFT JOIN order_gross og ON og.order_id = po.id\nLEFT JOIN order_refunds orf ON orf.order_id = po.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"youssofal_4bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3009,"phase_s":84.6015653340146,"decode_s":83.2762271668762,"prefill_s":1.3216383749386296,"decode_tps":36.13276084146202,"wall_s":90.41939929197542,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":36.13276084146202,"correct_answer_tps":36.13276084146202,"outcome":"pass","final_answer":"{\"read_results\": [10, null, 30, null, 31, null, null, 70], \"final_lru_to_mru\": [\"F\", \"H\", \"G\"], \"hits\": 4, \"misses\": 4}"},{"model":"youssofal_4bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2579,"phase_s":81.54990187496878,"decode_s":80.51017791591585,"prefill_s":1.0358077499549836,"decode_tps":32.03321700137697,"wall_s":87.47647341701668,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":32.03321700137697,"correct_answer_tps":32.03321700137697,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"youssofal_4bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1131,"phase_s":31.745057375053875,"decode_s":30.806621124967933,"prefill_s":0.9350159580353647,"decode_tps":36.712886993093676,"wall_s":39.19321029202547,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":36.712886993093676,"correct_answer_tps":36.712886993093676,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"coldfusion_8bit","label":"Qwen3.8 27B Cold Fusion 8-bit","original":{"attempts":8,"accepted_passes":8,"strict_passes":8,"delivered":8,"incomplete":0,"native_recorded_tokens":16746,"partial_observed_tokens":0,"native_phase_s":661.8058144159149,"partial_generation_s":0,"weighted_native_decode_tps":25.591936696462053,"known_wall_s":729.5296819159994,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":8,"strict_passes":8,"delivered":8,"incomplete":0,"native_recorded_tokens":16746,"partial_observed_tokens":0,"native_phase_s":661.8058144159149,"partial_generation_s":0,"weighted_native_decode_tps":25.591936696462053,"known_wall_s":729.5296819159994,"missing_wall_count":0},"original_tasks":[{"model":"coldfusion_8bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1810,"phase_s":66.95601420802996,"decode_s":66.20683083299082,"prefill_s":0.7457597091561183,"decode_tps":27.338568803660035,"wall_s":75.3644319580635,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":27.338568803660035,"correct_answer_tps":27.338568803660035,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"coldfusion_8bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2020,"phase_s":93.48123129201122,"decode_s":92.87716033309698,"prefill_s":0.6012886660173535,"decode_tps":21.74915762664816,"wall_s":102.50164637505077,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":21.74915762664816,"correct_answer_tps":21.74915762664816,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n by_user = {}\n\n for record in records:\n user = record[\"user\"]\n start = record[\"start\"]\n end = record[\"end\"]\n\n if end < start:\n raise ValueError(\"end must be greater than or equal to start\")\n\n by_user.setdefault(user, []).append((start, end))\n\n result = {}\n\n for user in sorted(by_user):\n intervals = sorted(by_user[user])\n total = 0\n current_start = None\n current_end = None\n\n for start, end in intervals:\n if current_start is None:\n current_start, current_end = start, end\n elif start > current_end:\n total += current_end - current_start\n current_start, current_end = start, end\n elif end > current_end:\n current_end = end\n\n if current_start is not None:\n total += current_end - current_start\n\n result[user] = total\n\n return result\n```"},{"model":"coldfusion_8bit","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3221,"phase_s":151.68291616695933,"decode_s":151.07175004296005,"prefill_s":0.6074654170079157,"decode_tps":21.3209948192435,"wall_s":162.7887031249702,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":21.3209948192435,"correct_answer_tps":21.3209948192435,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n nodes = set(graph)\n for prereqs in graph.values():\n nodes.update(prereqs)\n\n remaining = {node: set(graph.get(node, [])) for node in nodes}\n dependents = {node: [] for node in nodes}\n\n for node, prereqs in graph.items():\n for prereq in set(prereqs):\n dependents[prereq].append(node)\n\n available = [node for node in nodes if not remaining[node]]\n batches = []\n processed = 0\n\n while available:\n batch = sorted(available)\n batches.append(batch)\n processed += len(batch)\n\n next_available = []\n for node in batch:\n for dependent in dependents[node]:\n remaining[dependent].discard(node)\n if not remaining[dependent]:\n next_available.append(dependent)\n\n available = next_available\n\n if processed != len(nodes):\n raise ValueError(\"Cycle detected\")\n\n return batches\n```"},{"model":"coldfusion_8bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2513,"phase_s":104.646811415907,"decode_s":103.91397745790891,"prefill_s":0.7287471670424566,"decode_tps":24.183464645243788,"wall_s":111.53216087492183,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":24.183464645243788,"correct_answer_tps":24.183464645243788,"outcome":"pass","final_answer":"```sql\nWITH order_gross AS (\n SELECT\n order_id,\n SUM(qty * unit_cents) AS gross_cents\n FROM items\n GROUP BY order_id\n),\norder_refunds AS (\n SELECT\n order_id,\n SUM(amount_cents) AS refund_cents\n FROM refunds\n WHERE status = 'settled'\n GROUP BY order_id\n)\nSELECT\n c.id AS customer_id,\n COUNT(o.id) AS paid_order_count,\n COALESCE(\n SUM(\n COALESCE(og.gross_cents, 0) -\n COALESCE(rfd.refund_cents, 0)\n ),\n 0\n ) AS net_cents\nFROM customers AS c\nLEFT JOIN orders AS o\n ON o.customer_id = c.id\n AND o.status = 'paid'\nLEFT JOIN order_gross AS og\n ON og.order_id = o.id\nLEFT JOIN order_refunds AS rfd\n ON rfd.order_id = o.id\nGROUP BY c.id\nORDER BY c.id;\n```"},{"model":"coldfusion_8bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2826,"phase_s":96.2168834169861,"decode_s":94.80659083405044,"prefill_s":1.4042902080109343,"decode_tps":29.80805422005558,"wall_s":105.52070533391088,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":29.80805422005558,"correct_answer_tps":29.80805422005558,"outcome":"pass","final_answer":"{\"read_results\":[10,null,30,null,31,null,null,70],\"final_lru_to_mru\":[\"F\",\"H\",\"G\"],\"hits\":4,\"misses\":4}"},{"model":"coldfusion_8bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1388,"phase_s":49.07465012499597,"decode_s":47.96061983401887,"prefill_s":1.1106735409703106,"decode_tps":28.940409961413383,"wall_s":57.27972425008193,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.940409961413383,"correct_answer_tps":28.940409961413383,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"coldfusion_8bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":975,"phase_s":32.15250995801762,"decode_s":31.145683915005066,"prefill_s":1.0032657079864293,"decode_tps":31.304498005589593,"wall_s":39.20095420803409,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.304498005589593,"correct_answer_tps":31.304498005589593,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"},{"model":"coldfusion_8bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1993,"phase_s":67.59479783300776,"decode_s":66.36411316692829,"prefill_s":1.2277590000303462,"decode_tps":30.03129108328364,"wall_s":75.34135579096619,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.03129108328364,"correct_answer_tps":30.03129108328364,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"}],"selected_latest_tasks":[{"model":"coldfusion_8bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1810,"phase_s":66.95601420802996,"decode_s":66.20683083299082,"prefill_s":0.7457597091561183,"decode_tps":27.338568803660035,"wall_s":75.3644319580635,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":27.338568803660035,"correct_answer_tps":27.338568803660035,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"coldfusion_8bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2020,"phase_s":93.48123129201122,"decode_s":92.87716033309698,"prefill_s":0.6012886660173535,"decode_tps":21.74915762664816,"wall_s":102.50164637505077,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":21.74915762664816,"correct_answer_tps":21.74915762664816,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n by_user = {}\n\n for record in records:\n user = record[\"user\"]\n start = record[\"start\"]\n end = record[\"end\"]\n\n if end < start:\n raise ValueError(\"end must be greater than or equal to start\")\n\n by_user.setdefault(user, []).append((start, end))\n\n result = {}\n\n for user in sorted(by_user):\n intervals = sorted(by_user[user])\n total = 0\n current_start = None\n current_end = None\n\n for start, end in intervals:\n if current_start is None:\n current_start, current_end = start, end\n elif start > current_end:\n total += current_end - current_start\n current_start, current_end = start, end\n elif end > current_end:\n current_end = end\n\n if current_start is not None:\n total += current_end - current_start\n\n result[user] = total\n\n return result\n```"},{"model":"coldfusion_8bit","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3221,"phase_s":151.68291616695933,"decode_s":151.07175004296005,"prefill_s":0.6074654170079157,"decode_tps":21.3209948192435,"wall_s":162.7887031249702,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":21.3209948192435,"correct_answer_tps":21.3209948192435,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n nodes = set(graph)\n for prereqs in graph.values():\n nodes.update(prereqs)\n\n remaining = {node: set(graph.get(node, [])) for node in nodes}\n dependents = {node: [] for node in nodes}\n\n for node, prereqs in graph.items():\n for prereq in set(prereqs):\n dependents[prereq].append(node)\n\n available = [node for node in nodes if not remaining[node]]\n batches = []\n processed = 0\n\n while available:\n batch = sorted(available)\n batches.append(batch)\n processed += len(batch)\n\n next_available = []\n for node in batch:\n for dependent in dependents[node]:\n remaining[dependent].discard(node)\n if not remaining[dependent]:\n next_available.append(dependent)\n\n available = next_available\n\n if processed != len(nodes):\n raise ValueError(\"Cycle detected\")\n\n return batches\n```"},{"model":"coldfusion_8bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2513,"phase_s":104.646811415907,"decode_s":103.91397745790891,"prefill_s":0.7287471670424566,"decode_tps":24.183464645243788,"wall_s":111.53216087492183,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":24.183464645243788,"correct_answer_tps":24.183464645243788,"outcome":"pass","final_answer":"```sql\nWITH order_gross AS (\n SELECT\n order_id,\n SUM(qty * unit_cents) AS gross_cents\n FROM items\n GROUP BY order_id\n),\norder_refunds AS (\n SELECT\n order_id,\n SUM(amount_cents) AS refund_cents\n FROM refunds\n WHERE status = 'settled'\n GROUP BY order_id\n)\nSELECT\n c.id AS customer_id,\n COUNT(o.id) AS paid_order_count,\n COALESCE(\n SUM(\n COALESCE(og.gross_cents, 0) -\n COALESCE(rfd.refund_cents, 0)\n ),\n 0\n ) AS net_cents\nFROM customers AS c\nLEFT JOIN orders AS o\n ON o.customer_id = c.id\n AND o.status = 'paid'\nLEFT JOIN order_gross AS og\n ON og.order_id = o.id\nLEFT JOIN order_refunds AS rfd\n ON rfd.order_id = o.id\nGROUP BY c.id\nORDER BY c.id;\n```"},{"model":"coldfusion_8bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2826,"phase_s":96.2168834169861,"decode_s":94.80659083405044,"prefill_s":1.4042902080109343,"decode_tps":29.80805422005558,"wall_s":105.52070533391088,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":29.80805422005558,"correct_answer_tps":29.80805422005558,"outcome":"pass","final_answer":"{\"read_results\":[10,null,30,null,31,null,null,70],\"final_lru_to_mru\":[\"F\",\"H\",\"G\"],\"hits\":4,\"misses\":4}"},{"model":"coldfusion_8bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1388,"phase_s":49.07465012499597,"decode_s":47.96061983401887,"prefill_s":1.1106735409703106,"decode_tps":28.940409961413383,"wall_s":57.27972425008193,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.940409961413383,"correct_answer_tps":28.940409961413383,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"coldfusion_8bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":975,"phase_s":32.15250995801762,"decode_s":31.145683915005066,"prefill_s":1.0032657079864293,"decode_tps":31.304498005589593,"wall_s":39.20095420803409,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":31.304498005589593,"correct_answer_tps":31.304498005589593,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"},{"model":"coldfusion_8bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1993,"phase_s":67.59479783300776,"decode_s":66.36411316692829,"prefill_s":1.2277590000303462,"decode_tps":30.03129108328364,"wall_s":75.34135579096619,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.03129108328364,"correct_answer_tps":30.03129108328364,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"youssofal_8bit","label":"Qwen3.8 27B Youssofal 8-bit","original":{"attempts":8,"accepted_passes":8,"strict_passes":8,"delivered":8,"incomplete":0,"native_recorded_tokens":13972,"partial_observed_tokens":0,"native_phase_s":501.0631732508773,"partial_generation_s":0,"weighted_native_decode_tps":28.304579969483413,"known_wall_s":576.3275173329748,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":8,"strict_passes":8,"delivered":8,"incomplete":0,"native_recorded_tokens":13972,"partial_observed_tokens":0,"native_phase_s":501.0631732508773,"partial_generation_s":0,"weighted_native_decode_tps":28.304579969483413,"known_wall_s":576.3275173329748,"missing_wall_count":0},"original_tasks":[{"model":"youssofal_8bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1795,"phase_s":73.86883666692302,"decode_s":73.12398891686462,"prefill_s":0.7414654999738559,"decode_tps":24.547347957737276,"wall_s":81.37314337491989,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":24.547347957737276,"correct_answer_tps":24.547347957737276,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"youssofal_8bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1236,"phase_s":47.0337170829298,"decode_s":46.4278222079156,"prefill_s":0.6021419570315629,"decode_tps":26.621968061841834,"wall_s":57.74703408393543,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.621968061841834,"correct_answer_tps":26.621968061841834,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for record in records:\n user = record['user']\n start = record['start']\n end = record['end']\n if end < start:\n raise ValueError\n if user not in user_intervals:\n user_intervals[user] = []\n user_intervals[user].append((start, end))\n\n result = {}\n for user in sorted(user_intervals):\n intervals = user_intervals[user]\n intervals.sort()\n total = 0\n cur_start, cur_end = intervals[0]\n for s, e in intervals[1:]:\n if s <= cur_end:\n if e > cur_end:\n cur_end = e\n else:\n total += cur_end - cur_start\n cur_start, cur_end = s, e\n total += cur_end - cur_start\n result[user] = total\n return result\n```"},{"model":"youssofal_8bit","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1693,"phase_s":63.612838334054686,"decode_s":63.01779387507122,"prefill_s":0.5913187509868294,"decode_tps":26.86542793542194,"wall_s":72.34935408306774,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.86542793542194,"correct_answer_tps":26.86542793542194,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n all_nodes = set(graph.keys())\n for prereqs in graph.values():\n for p in prereqs:\n all_nodes.add(p)\n\n prereqs = {}\n for node in all_nodes:\n if node in graph:\n prereqs[node] = set(graph[node])\n else:\n prereqs[node] = set()\n\n remaining = set(all_nodes)\n processed = set()\n batches = []\n\n while remaining:\n available = set()\n for node in remaining:\n if prereqs[node].issubset(processed):\n available.add(node)\n if not available:\n raise ValueError(\"Cycle detected\")\n batches.append(sorted(available))\n processed.update(available)\n remaining -= available\n\n return batches\n```"},{"model":"youssofal_8bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1202,"phase_s":45.76258362503722,"decode_s":45.0304620009847,"prefill_s":0.7287494170013815,"decode_tps":26.69304170083166,"wall_s":63.399955083034,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.69304170083166,"correct_answer_tps":26.69304170083166,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT id, customer_id\n FROM orders\n WHERE status = 'paid'\n),\norder_gross AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.id\n GROUP BY po.id\n),\norder_refunds AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(r.amount_cents), 0) AS total_refunds\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.id AND r.status = 'settled'\n GROUP BY po.id\n),\norder_net AS (\n SELECT po.id AS order_id,\n po.customer_id,\n og.gross - orf.total_refunds AS net\n FROM paid_orders po\n JOIN order_gross og ON og.order_id = po.id\n JOIN order_refunds orf ON orf.order_id = po.id\n)\nSELECT c.id AS customer_id,\n COUNT(onet.order_id) AS paid_order_count,\n COALESCE(SUM(onet.net), 0) AS net_cents\nFROM customers c\nLEFT JOIN order_net onet ON onet.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"youssofal_8bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3037,"phase_s":100.55412712495308,"decode_s":99.15918583294842,"prefill_s":1.3914942492265254,"decode_tps":30.627520531646717,"wall_s":108.52391225006431,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.627520531646717,"correct_answer_tps":30.627520531646717,"outcome":"pass","final_answer":"{\"read_results\": [10, null, 30, null, 31, null, null, 70], \"final_lru_to_mru\": [\"F\", \"H\", \"G\"], \"hits\": 4, \"misses\": 4}"},{"model":"youssofal_8bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1997,"phase_s":69.8764549579937,"decode_s":68.7656158329919,"prefill_s":1.107214249088429,"decode_tps":29.04067644576947,"wall_s":78.43033837492112,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":29.04067644576947,"correct_answer_tps":29.04067644576947,"outcome":"pass","final_answer":"[\n {\"id\": \"S-17\", \"status\": \"departed\", \"units\": 10, \"departed_utc\": \"2026-05-04T22:10:00Z\", \"evidence\": [3, 5]},\n {\"id\": \"S-18\", \"status\": \"planned\", \"units\": 9, \"departed_utc\": null, \"evidence\": [7]},\n {\"id\": \"S-19\", \"status\": \"departed\", \"units\": 5, \"departed_utc\": \"2026-05-05T02:50:00Z\", \"evidence\": [6]},\n {\"id\": \"S-20\", \"status\": \"cancelled\", \"units\": 0, \"departed_utc\": null, \"evidence\": [8]}\n]"},{"model":"youssofal_8bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1155,"phase_s":38.70995404198766,"decode_s":37.70080991683062,"prefill_s":1.0055743341799825,"decode_tps":30.63594661621256,"wall_s":45.195965167018585,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.63594661621256,"correct_answer_tps":30.63594661621256,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"},{"model":"youssofal_8bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1857,"phase_s":61.64466141699813,"decode_s":60.404681416926906,"prefill_s":1.2364426670828834,"decode_tps":30.742650344971807,"wall_s":69.30781491601374,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.742650344971807,"correct_answer_tps":30.742650344971807,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"}],"selected_latest_tasks":[{"model":"youssofal_8bit","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1795,"phase_s":73.86883666692302,"decode_s":73.12398891686462,"prefill_s":0.7414654999738559,"decode_tps":24.547347957737276,"wall_s":81.37314337491989,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":24.547347957737276,"correct_answer_tps":24.547347957737276,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"youssofal_8bit","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1236,"phase_s":47.0337170829298,"decode_s":46.4278222079156,"prefill_s":0.6021419570315629,"decode_tps":26.621968061841834,"wall_s":57.74703408393543,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.621968061841834,"correct_answer_tps":26.621968061841834,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for record in records:\n user = record['user']\n start = record['start']\n end = record['end']\n if end < start:\n raise ValueError\n if user not in user_intervals:\n user_intervals[user] = []\n user_intervals[user].append((start, end))\n\n result = {}\n for user in sorted(user_intervals):\n intervals = user_intervals[user]\n intervals.sort()\n total = 0\n cur_start, cur_end = intervals[0]\n for s, e in intervals[1:]:\n if s <= cur_end:\n if e > cur_end:\n cur_end = e\n else:\n total += cur_end - cur_start\n cur_start, cur_end = s, e\n total += cur_end - cur_start\n result[user] = total\n return result\n```"},{"model":"youssofal_8bit","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1693,"phase_s":63.612838334054686,"decode_s":63.01779387507122,"prefill_s":0.5913187509868294,"decode_tps":26.86542793542194,"wall_s":72.34935408306774,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.86542793542194,"correct_answer_tps":26.86542793542194,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n all_nodes = set(graph.keys())\n for prereqs in graph.values():\n for p in prereqs:\n all_nodes.add(p)\n\n prereqs = {}\n for node in all_nodes:\n if node in graph:\n prereqs[node] = set(graph[node])\n else:\n prereqs[node] = set()\n\n remaining = set(all_nodes)\n processed = set()\n batches = []\n\n while remaining:\n available = set()\n for node in remaining:\n if prereqs[node].issubset(processed):\n available.add(node)\n if not available:\n raise ValueError(\"Cycle detected\")\n batches.append(sorted(available))\n processed.update(available)\n remaining -= available\n\n return batches\n```"},{"model":"youssofal_8bit","task":"sql","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1202,"phase_s":45.76258362503722,"decode_s":45.0304620009847,"prefill_s":0.7287494170013815,"decode_tps":26.69304170083166,"wall_s":63.399955083034,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.69304170083166,"correct_answer_tps":26.69304170083166,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT id, customer_id\n FROM orders\n WHERE status = 'paid'\n),\norder_gross AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.id\n GROUP BY po.id\n),\norder_refunds AS (\n SELECT po.id AS order_id,\n COALESCE(SUM(r.amount_cents), 0) AS total_refunds\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.id AND r.status = 'settled'\n GROUP BY po.id\n),\norder_net AS (\n SELECT po.id AS order_id,\n po.customer_id,\n og.gross - orf.total_refunds AS net\n FROM paid_orders po\n JOIN order_gross og ON og.order_id = po.id\n JOIN order_refunds orf ON orf.order_id = po.id\n)\nSELECT c.id AS customer_id,\n COUNT(onet.order_id) AS paid_order_count,\n COALESCE(SUM(onet.net), 0) AS net_cents\nFROM customers c\nLEFT JOIN order_net onet ON onet.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"youssofal_8bit","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3037,"phase_s":100.55412712495308,"decode_s":99.15918583294842,"prefill_s":1.3914942492265254,"decode_tps":30.627520531646717,"wall_s":108.52391225006431,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.627520531646717,"correct_answer_tps":30.627520531646717,"outcome":"pass","final_answer":"{\"read_results\": [10, null, 30, null, 31, null, null, 70], \"final_lru_to_mru\": [\"F\", \"H\", \"G\"], \"hits\": 4, \"misses\": 4}"},{"model":"youssofal_8bit","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1997,"phase_s":69.8764549579937,"decode_s":68.7656158329919,"prefill_s":1.107214249088429,"decode_tps":29.04067644576947,"wall_s":78.43033837492112,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":29.04067644576947,"correct_answer_tps":29.04067644576947,"outcome":"pass","final_answer":"[\n {\"id\": \"S-17\", \"status\": \"departed\", \"units\": 10, \"departed_utc\": \"2026-05-04T22:10:00Z\", \"evidence\": [3, 5]},\n {\"id\": \"S-18\", \"status\": \"planned\", \"units\": 9, \"departed_utc\": null, \"evidence\": [7]},\n {\"id\": \"S-19\", \"status\": \"departed\", \"units\": 5, \"departed_utc\": \"2026-05-05T02:50:00Z\", \"evidence\": [6]},\n {\"id\": \"S-20\", \"status\": \"cancelled\", \"units\": 0, \"departed_utc\": null, \"evidence\": [8]}\n]"},{"model":"youssofal_8bit","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1155,"phase_s":38.70995404198766,"decode_s":37.70080991683062,"prefill_s":1.0055743341799825,"decode_tps":30.63594661621256,"wall_s":45.195965167018585,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.63594661621256,"correct_answer_tps":30.63594661621256,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"},{"model":"youssofal_8bit","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1857,"phase_s":61.64466141699813,"decode_s":60.404681416926906,"prefill_s":1.2364426670828834,"decode_tps":30.742650344971807,"wall_s":69.30781491601374,"wall_kind":"fresh command","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":30.742650344971807,"correct_answer_tps":30.742650344971807,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"flash","label":"Flash-Next 3.3 bpw","original":{"attempts":8,"accepted_passes":5,"strict_passes":5,"delivered":5,"incomplete":3,"native_recorded_tokens":23195,"partial_observed_tokens":0,"native_phase_s":853.577261,"partial_generation_s":0,"weighted_native_decode_tps":28.01715355692253,"known_wall_s":854.3108944190899,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":6,"strict_passes":6,"delivered":6,"incomplete":2,"native_recorded_tokens":32216,"partial_observed_tokens":0,"native_phase_s":1190.435072,"partial_generation_s":0,"weighted_native_decode_tps":27.623303036619433,"known_wall_s":1190.9909916670294,"missing_wall_count":0},"original_tasks":[{"model":"flash","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1760,"phase_s":41.417153,"decode_s":38.294593,"prefill_s":3.12256,"decode_tps":45.959,"wall_s":41.53180604195222,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":45.959,"correct_answer_tps":45.959,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"flash","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3356,"phase_s":135.066993,"decode_s":133.772786,"prefill_s":1.294207,"decode_tps":25.087,"wall_s":135.0948029999854,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":25.087,"correct_answer_tps":25.087,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"flash","task":"intervals","phase":"original","budget":4096,"passed":false,"score":0.0,"content_passed":false,"tokens":4096,"phase_s":163.21846299999999,"decode_s":161.57925899999998,"prefill_s":1.6392039999999999,"decode_tps":25.35,"wall_s":163.30157554207835,"wall_kind":"resident server HTTP","finish":"length","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"flash","task":"batches","phase":"original","budget":4096,"passed":false,"score":0.0,"content_passed":false,"tokens":4096,"phase_s":161.12756,"decode_s":158.724965,"prefill_s":2.402595,"decode_tps":25.806,"wall_s":161.27109291707166,"wall_kind":"resident server HTTP","finish":"length","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"flash","task":"sql","phase":"original","budget":4096,"passed":false,"score":0.0,"content_passed":false,"tokens":4096,"phase_s":153.54538300000002,"decode_s":152.064022,"prefill_s":1.4813610000000001,"decode_tps":26.936,"wall_s":153.67774195899256,"wall_kind":"resident server HTTP","finish":"length","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"flash","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2673,"phase_s":100.614976,"decode_s":93.669215,"prefill_s":6.945761,"decode_tps":28.537,"wall_s":100.63117175002117,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.537,"correct_answer_tps":28.537,"outcome":"pass","final_answer":"{\"read_results\":[10,null,30,null,31,null,null,70],\"final_lru_to_mru\":[\"F\",\"H\",\"G\"],\"hits\":4,\"misses\":4}"},{"model":"flash","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1937,"phase_s":61.924768,"decode_s":59.140711,"prefill_s":2.784057,"decode_tps":32.752,"wall_s":62.02866533398628,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":32.752,"correct_answer_tps":32.752,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"flash","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1181,"phase_s":36.661964999999995,"decode_s":30.640121,"prefill_s":6.021844,"decode_tps":38.544,"wall_s":36.77403787500225,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":38.544,"correct_answer_tps":38.544,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"}],"selected_latest_tasks":[{"model":"flash","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1760,"phase_s":41.417153,"decode_s":38.294593,"prefill_s":3.12256,"decode_tps":45.959,"wall_s":41.53180604195222,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":45.959,"correct_answer_tps":45.959,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"flash","task":"schedule","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3356,"phase_s":135.066993,"decode_s":133.772786,"prefill_s":1.294207,"decode_tps":25.087,"wall_s":135.0948029999854,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":25.087,"correct_answer_tps":25.087,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"flash","task":"intervals","phase":"retry","budget":16384,"passed":false,"score":0.0,"content_passed":false,"tokens":16384,"phase_s":622.192888,"decode_s":620.742481,"prefill_s":1.450407,"decode_tps":26.394,"wall_s":622.2260427920846,"wall_kind":"resident server HTTP","finish":"length","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"flash","task":"batches","phase":"retry","budget":16384,"passed":false,"score":0.0,"content_passed":false,"tokens":2735,"phase_s":110.03713,"decode_s":108.912676,"prefill_s":1.1244539999999998,"decode_tps":25.112,"wall_s":110.05413808301091,"wall_kind":"resident server HTTP","finish":"repetition_loop","strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"flash","task":"sql","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":2190,"phase_s":82.51919900000001,"decode_s":81.089104,"prefill_s":1.4300950000000001,"decode_tps":27.007,"wall_s":82.65032679098658,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":27.007,"correct_answer_tps":27.007,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT id, customer_id\n FROM orders\n WHERE status = 'paid'\n),\norder_gross AS (\n SELECT\n po.id AS order_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross_cents\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.id\n GROUP BY po.id\n),\norder_refunds AS (\n SELECT\n po.id AS order_id,\n COALESCE(SUM(r.amount_cents), 0) AS refund_cents\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.id AND r.status = 'settled'\n GROUP BY po.id\n),\norder_net AS (\n SELECT\n po.id AS order_id,\n po.customer_id,\n g.gross_cents - r.refund_cents AS net_cents\n FROM paid_orders po\n JOIN order_gross g ON g.order_id = po.id\n JOIN order_refunds r ON r.order_id = po.id\n)\nSELECT\n c.id AS customer_id,\n COUNT(n.order_id) AS paid_order_count,\n COALESCE(SUM(n.net_cents), 0) AS net_cents\nFROM customers c\nLEFT JOIN order_net n ON n.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"flash","task":"cache","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2673,"phase_s":100.614976,"decode_s":93.669215,"prefill_s":6.945761,"decode_tps":28.537,"wall_s":100.63117175002117,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.537,"correct_answer_tps":28.537,"outcome":"pass","final_answer":"{\"read_results\":[10,null,30,null,31,null,null,70],\"final_lru_to_mru\":[\"F\",\"H\",\"G\"],\"hits\":4,\"misses\":4}"},{"model":"flash","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1937,"phase_s":61.924768,"decode_s":59.140711,"prefill_s":2.784057,"decode_tps":32.752,"wall_s":62.02866533398628,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":32.752,"correct_answer_tps":32.752,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"flash","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1181,"phase_s":36.661964999999995,"decode_s":30.640121,"prefill_s":6.021844,"decode_tps":38.544,"wall_s":36.77403787500225,"wall_kind":"resident server HTTP","finish":"stop","strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":38.544,"correct_answer_tps":38.544,"outcome":"pass","final_answer":"{\"A\":{\"eligible\":false,\"credit\":0},\"B\":{\"eligible\":true,\"credit\":8.5},\"C\":{\"eligible\":false,\"credit\":0},\"D\":{\"eligible\":null,\"credit\":null},\"E\":{\"eligible\":true,\"credit\":null},\"F\":{\"eligible\":false,\"credit\":0}}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"gemma","label":"Gemma 4 31B Q4 + Q6 assistant","original":{"attempts":8,"accepted_passes":3,"strict_passes":0,"delivered":3,"incomplete":5,"native_recorded_tokens":27835,"partial_observed_tokens":0,"native_phase_s":1437.012250206899,"partial_generation_s":0,"weighted_native_decode_tps":19.540178603589446,"known_wall_s":1298.3525470849127,"missing_wall_count":1},"selected_latest":{"attempts":8,"accepted_passes":5,"strict_passes":1,"delivered":5,"incomplete":3,"native_recorded_tokens":17615,"partial_observed_tokens":84224,"native_phase_s":702.1706234160811,"partial_generation_s":3791.429680415895,"weighted_native_decode_tps":25.34819427053376,"known_wall_s":4553.955638807383,"missing_wall_count":0},"original_tasks":[{"model":"gemma","task":"invoice","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":true,"tokens":3889,"phase_s":171.68472404091153,"decode_s":169.97437399893533,"prefill_s":1.7103500419761986,"decode_tps":22.87991953436675,"wall_s":null,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.billed keys","passed":true},{"name":"$.billed.I1","passed":true,"expected":"89.54","actual":"89.54"},{"name":"$.billed.I2","passed":true,"expected":"70.55","actual":"70.55"},{"name":"$.billed.I3","passed":true,"expected":"70.98","actual":"70.98"},{"name":"$.billed.I4","passed":true,"expected":"191.45","actual":"191.45"},{"name":"$.net_by_customer keys","passed":true},{"name":"$.net_by_customer.Ada","passed":true,"expected":"129.43","actual":"129.43"},{"name":"$.net_by_customer.Ben","passed":true,"expected":"70.55","actual":"70.55"},{"name":"$.net_by_customer.Cy","passed":true,"expected":"166.45","actual":"166.45"},{"name":"$.grand_net","passed":true,"expected":"366.43","actual":"366.43"}]},"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":22.87991953436675,"correct_answer_tps":22.87991953436675,"outcome":"pass","final_answer":"```json\n{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}\n```"},{"model":"gemma","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":203.32809608301613,"decode_s":201.92546800000127,"prefill_s":1.4026280830148607,"decode_tps":20.284712179050015,"wall_s":208.25011570798233,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"intervals","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":251.29115433304105,"decode_s":249.81595658301376,"prefill_s":1.4751977500272915,"decode_tps":16.39607035525331,"wall_s":256.1692023749929,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"batches","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":171.47159033396747,"decode_s":170.06875295890495,"prefill_s":1.4028373750625178,"decode_tps":24.08437722236811,"wall_s":174.65989408397581,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"sql","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":222.8421667079674,"decode_s":221.48545454104897,"prefill_s":1.3567121669184417,"decode_tps":18.49331374147131,"wall_s":228.7846186250681,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":224.33970024995506,"decode_s":222.27178116701543,"prefill_s":2.067919082939625,"decode_tps":18.427890299408983,"wall_s":228.77876154205296,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"extract","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":true,"tokens":2371,"phase_s":133.1791450829478,"decode_s":131.5179133749334,"prefill_s":1.661231708014384,"decode_tps":18.02796242091155,"wall_s":138.49183116690256,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ length","passed":true},{"name":"$[0] keys","passed":true},{"name":"$[0].id","passed":true,"expected":"S-17","actual":"S-17"},{"name":"$[0].status","passed":true,"expected":"departed","actual":"departed"},{"name":"$[0].units","passed":true,"expected":10,"actual":10},{"name":"$[0].departed_utc","passed":true,"expected":"2026-05-04T22:10:00Z","actual":"2026-05-04T22:10:00Z"},{"name":"$[0].evidence length","passed":true},{"name":"$[0].evidence[0]","passed":true,"expected":"L3","actual":"L3"},{"name":"$[0].evidence[1]","passed":true,"expected":"L5","actual":"L5"},{"name":"$[1] keys","passed":true},{"name":"$[1].id","passed":true,"expected":"S-18","actual":"S-18"},{"name":"$[1].status","passed":true,"expected":"planned","actual":"planned"},{"name":"$[1].units","passed":true,"expected":9,"actual":9},{"name":"$[1].departed_utc","passed":true,"expected":null,"actual":null},{"name":"$[1].evidence length","passed":true},{"name":"$[1].evidence[0]","passed":true,"expected":"L7","actual":"L7"},{"name":"$[2] keys","passed":true},{"name":"$[2].id","passed":true,"expected":"S-19","actual":"S-19"},{"name":"$[2].status","passed":true,"expected":"departed","actual":"departed"},{"name":"$[2].units","passed":true,"expected":5,"actual":5},{"name":"$[2].departed_utc","passed":true,"expected":"2026-05-05T02:50:00Z","actual":"2026-05-05T02:50:00Z"},{"name":"$[2].evidence length","passed":true},{"name":"$[2].evidence[0]","passed":true,"expected":"L6","actual":"L6"},{"name":"$[3] keys","passed":true},{"name":"$[3].id","passed":true,"expected":"S-20","actual":"S-20"},{"name":"$[3].status","passed":true,"expected":"cancelled","actual":"cancelled"},{"name":"$[3].units","passed":true,"expected":0,"actual":0},{"name":"$[3].departed_utc","passed":true,"expected":null,"actual":null},{"name":"$[3].evidence length","passed":true},{"name":"$[3].evidence[0]","passed":true,"expected":"L8","actual":"L8"}]},"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":18.02796242091155,"correct_answer_tps":18.02796242091155,"outcome":"pass","final_answer":"```json\n[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\n \"L3\",\n \"L5\"\n ]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\n \"L7\"\n ]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [\n \"L6\"\n ]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\n \"L8\"\n ]\n }\n]\n```"},{"model":"gemma","task":"policy","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":true,"tokens":1095,"phase_s":58.87567337509245,"decode_s":57.441096666036174,"prefill_s":1.434576709056273,"decode_tps":19.063006515463215,"wall_s":63.218123583937995,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.A keys","passed":true},{"name":"$.A.eligible","passed":true,"expected":false,"actual":false},{"name":"$.A.credit","passed":true,"expected":0,"actual":0},{"name":"$.B keys","passed":true},{"name":"$.B.eligible","passed":true,"expected":true,"actual":true},{"name":"$.B.credit","passed":true,"expected":8.5,"actual":8.5},{"name":"$.C keys","passed":true},{"name":"$.C.eligible","passed":true,"expected":false,"actual":false},{"name":"$.C.credit","passed":true,"expected":0,"actual":0},{"name":"$.D keys","passed":true},{"name":"$.D.eligible","passed":true,"expected":null,"actual":null},{"name":"$.D.credit","passed":true,"expected":null,"actual":null},{"name":"$.E keys","passed":true},{"name":"$.E.eligible","passed":true,"expected":true,"actual":true},{"name":"$.E.credit","passed":true,"expected":null,"actual":null},{"name":"$.F keys","passed":true},{"name":"$.F.eligible","passed":true,"expected":false,"actual":false},{"name":"$.F.credit","passed":true,"expected":0,"actual":0}]},"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":19.063006515463215,"correct_answer_tps":19.063006515463215,"outcome":"pass","final_answer":"```json\n{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}\n```"}],"selected_latest_tasks":[{"model":"gemma","task":"invoice","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":3889,"phase_s":113.75483020802494,"decode_s":112.0442894580774,"prefill_s":1.7105407499475405,"decode_tps":34.7094887103114,"wall_s":120.58322929101996,"wall_kind":"fresh command","interrupted":false,"finish":"stop","partial_generation_s":null,"partial_observed_tps":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.billed keys","passed":true},{"name":"$.billed.I1","passed":true,"expected":"89.54","actual":"89.54"},{"name":"$.billed.I2","passed":true,"expected":"70.55","actual":"70.55"},{"name":"$.billed.I3","passed":true,"expected":"70.98","actual":"70.98"},{"name":"$.billed.I4","passed":true,"expected":"191.45","actual":"191.45"},{"name":"$.net_by_customer keys","passed":true},{"name":"$.net_by_customer.Ada","passed":true,"expected":"129.43","actual":"129.43"},{"name":"$.net_by_customer.Ben","passed":true,"expected":"70.55","actual":"70.55"},{"name":"$.net_by_customer.Cy","passed":true,"expected":"166.45","actual":"166.45"},{"name":"$.grand_net","passed":true,"expected":"366.43","actual":"366.43"}]},"final_text_present":true,"delivered_final":true,"display_tps":34.7094887103114,"correct_answer_tps":34.7094887103114,"outcome":"pass","final_answer":"```json\n{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}\n```"},{"model":"gemma","task":"schedule","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":5361,"phase_s":228.9141681670444,"decode_s":227.52758808410726,"prefill_s":1.3865800829371437,"decode_tps":23.56197789086687,"wall_s":231.89639829203952,"wall_kind":"fresh command","interrupted":false,"finish":"stop","partial_generation_s":null,"partial_observed_tps":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.makespan","passed":true,"expected":11,"actual":11},{"name":"$.starts keys","passed":true},{"name":"$.starts.A","passed":true,"expected":0,"actual":0},{"name":"$.starts.B","passed":true,"expected":1,"actual":1},{"name":"$.starts.C","passed":true,"expected":3,"actual":3},{"name":"$.starts.D","passed":true,"expected":3,"actual":3},{"name":"$.starts.E","passed":true,"expected":7,"actual":7},{"name":"$.starts.F","passed":true,"expected":10,"actual":10}]},"final_text_present":true,"delivered_final":true,"display_tps":23.56197789086687,"correct_answer_tps":23.56197789086687,"outcome":"pass","final_answer":"```json\n{\"makespan\": 11, \"starts\": {\"A\": 0, \"B\": 1, \"C\": 3, \"D\": 3, \"E\": 7, \"F\": 10}}\n```"},{"model":"gemma","task":"intervals","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":18688,"phase_s":null,"decode_s":null,"prefill_s":null,"decode_tps":null,"wall_s":794.9099781513214,"wall_kind":"fresh command","interrupted":true,"finish":"sustained repeated event sequence (exploratory multiblock guard)","partial_generation_s":786.6279608749319,"partial_observed_tps":23.75710110687415,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"batches","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":32768,"phase_s":null,"decode_s":null,"prefill_s":null,"decode_tps":null,"wall_s":1663.9728689193726,"wall_kind":"approximate: first resource sample to manual-stop receipt; includes stop/poll delay","interrupted":true,"finish":"32,768-token evidence boundary reached without final answer; no exact-paragraph loop guard","partial_generation_s":1643.1116599580273,"partial_observed_tps":19.942649546310836,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"sql","phase":"gemma_large_budget","budget":131072,"passed":true,"score":100.0,"tokens":4899,"phase_s":218.55173329100944,"decode_s":217.19829758291598,"prefill_s":1.353435708093457,"decode_tps":22.555425408570684,"wall_s":223.20700791699346,"wall_kind":"fresh command","interrupted":false,"finish":"stop","partial_generation_s":null,"partial_observed_tps":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"final_text_present":true,"delivered_final":true,"display_tps":22.555425408570684,"correct_answer_tps":22.555425408570684,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT id, customer_id\n FROM orders\n WHERE status = 'paid'\n),\norder_gross AS (\n SELECT order_id, SUM(qty * unit_cents) AS gross\n FROM items\n GROUP BY order_id\n),\norder_refunds AS (\n SELECT order_id, SUM(amount_cents) AS refund_sum\n FROM refunds\n WHERE status = 'settled'\n GROUP BY order_id\n)\nSELECT\n c.id AS customer_id,\n COUNT(po.id) AS paid_order_count,\n SUM(COALESCE(og.gross, 0) - COALESCE(orf.refund_sum, 0)) AS net_cents\nFROM customers c\nLEFT JOIN paid_orders po ON c.id = po.customer_id\nLEFT JOIN order_gross og ON po.id = og.order_id\nLEFT JOIN order_refunds orf ON po.id = orf.order_id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"gemma","task":"cache","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":32768,"phase_s":null,"decode_s":null,"prefill_s":null,"decode_tps":null,"wall_s":1368.6846199035645,"wall_kind":"fresh command","interrupted":true,"finish":"sustained repeated event sequence (exploratory multiblock guard)","partial_generation_s":1361.6900595829356,"partial_observed_tps":24.064213268940456,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"gemma","task":"extract","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":2371,"phase_s":97.27469087496866,"decode_s":95.82975770789199,"prefill_s":1.4449331670766696,"decode_tps":24.741792703131694,"wall_s":102.47479850007221,"wall_kind":"fresh command","interrupted":false,"finish":"stop","partial_generation_s":null,"partial_observed_tps":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ length","passed":true},{"name":"$[0] keys","passed":true},{"name":"$[0].id","passed":true,"expected":"S-17","actual":"S-17"},{"name":"$[0].status","passed":true,"expected":"departed","actual":"departed"},{"name":"$[0].units","passed":true,"expected":10,"actual":10},{"name":"$[0].departed_utc","passed":true,"expected":"2026-05-04T22:10:00Z","actual":"2026-05-04T22:10:00Z"},{"name":"$[0].evidence length","passed":true},{"name":"$[0].evidence[0]","passed":true,"expected":"L3","actual":"L3"},{"name":"$[0].evidence[1]","passed":true,"expected":"L5","actual":"L5"},{"name":"$[1] keys","passed":true},{"name":"$[1].id","passed":true,"expected":"S-18","actual":"S-18"},{"name":"$[1].status","passed":true,"expected":"planned","actual":"planned"},{"name":"$[1].units","passed":true,"expected":9,"actual":9},{"name":"$[1].departed_utc","passed":true,"expected":null,"actual":null},{"name":"$[1].evidence length","passed":true},{"name":"$[1].evidence[0]","passed":true,"expected":"L7","actual":"L7"},{"name":"$[2] keys","passed":true},{"name":"$[2].id","passed":true,"expected":"S-19","actual":"S-19"},{"name":"$[2].status","passed":true,"expected":"departed","actual":"departed"},{"name":"$[2].units","passed":true,"expected":5,"actual":5},{"name":"$[2].departed_utc","passed":true,"expected":"2026-05-05T02:50:00Z","actual":"2026-05-05T02:50:00Z"},{"name":"$[2].evidence length","passed":true},{"name":"$[2].evidence[0]","passed":true,"expected":"L6","actual":"L6"},{"name":"$[3] keys","passed":true},{"name":"$[3].id","passed":true,"expected":"S-20","actual":"S-20"},{"name":"$[3].status","passed":true,"expected":"cancelled","actual":"cancelled"},{"name":"$[3].units","passed":true,"expected":0,"actual":0},{"name":"$[3].departed_utc","passed":true,"expected":null,"actual":null},{"name":"$[3].evidence length","passed":true},{"name":"$[3].evidence[0]","passed":true,"expected":"L8","actual":"L8"}]},"final_text_present":true,"delivered_final":true,"display_tps":24.741792703131694,"correct_answer_tps":24.741792703131694,"outcome":"pass","final_answer":"```json\n[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\n \"L3\",\n \"L5\"\n ]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\n \"L7\"\n ]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [\n \"L6\"\n ]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\n \"L8\"\n ]\n }\n]\n```"},{"model":"gemma","task":"policy","phase":"gemma_large_budget","budget":131072,"passed":false,"score":0,"tokens":1095,"phase_s":43.675200875033624,"decode_s":42.32136262499262,"prefill_s":1.353838250041008,"decode_tps":25.873458038266342,"wall_s":48.22673783299979,"wall_kind":"fresh command","interrupted":false,"finish":"stop","partial_generation_s":null,"partial_observed_tps":null,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.A keys","passed":true},{"name":"$.A.eligible","passed":true,"expected":false,"actual":false},{"name":"$.A.credit","passed":true,"expected":0,"actual":0},{"name":"$.B keys","passed":true},{"name":"$.B.eligible","passed":true,"expected":true,"actual":true},{"name":"$.B.credit","passed":true,"expected":8.5,"actual":8.5},{"name":"$.C keys","passed":true},{"name":"$.C.eligible","passed":true,"expected":false,"actual":false},{"name":"$.C.credit","passed":true,"expected":0,"actual":0},{"name":"$.D keys","passed":true},{"name":"$.D.eligible","passed":true,"expected":null,"actual":null},{"name":"$.D.credit","passed":true,"expected":null,"actual":null},{"name":"$.E keys","passed":true},{"name":"$.E.eligible","passed":true,"expected":true,"actual":true},{"name":"$.E.credit","passed":true,"expected":null,"actual":null},{"name":"$.F keys","passed":true},{"name":"$.F.eligible","passed":true,"expected":false,"actual":false},{"name":"$.F.credit","passed":true,"expected":0,"actual":0}]},"final_text_present":true,"delivered_final":true,"display_tps":25.873458038266342,"correct_answer_tps":25.873458038266342,"outcome":"pass","final_answer":"```json\n{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}\n```"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"qwen36","label":"Qwen3.6 35B-A3B Balance Q6","original":{"attempts":8,"accepted_passes":1,"strict_passes":1,"delivered":1,"incomplete":7,"native_recorded_tokens":31541,"partial_observed_tokens":0,"native_phase_s":522.6000848748954,"partial_generation_s":0,"weighted_native_decode_tps":60.702838746954214,"known_wall_s":569.6305442498997,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":6,"strict_passes":6,"delivered":8,"incomplete":0,"native_recorded_tokens":56946,"partial_observed_tokens":0,"native_phase_s":817.6951854999643,"partial_generation_s":0,"weighted_native_decode_tps":69.84365196805844,"known_wall_s":868.109372084029,"missing_wall_count":0},"original_tasks":[{"model":"qwen36","task":"invoice","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":66.10128491697833,"decode_s":65.47524750092998,"prefill_s":0.622788623906672,"decode_tps":62.55799185702692,"wall_s":72.34522874990944,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":66.89489600004163,"decode_s":66.59954691701569,"prefill_s":0.29209949797950685,"decode_tps":61.50191990200316,"wall_s":72.38996749999933,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"intervals","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":73.90775004192255,"decode_s":73.66351037588902,"prefill_s":0.2409490409772843,"decode_tps":55.60419234840961,"wall_s":78.32637937495019,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"batches","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":57.902875541942194,"decode_s":57.66852762491908,"prefill_s":0.23129908286500722,"decode_tps":71.02660963776856,"wall_s":63.32195749995299,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"sql","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":71.36474254098721,"decode_s":71.00083866599016,"prefill_s":0.3608446669531986,"decode_tps":57.68945940580853,"wall_s":78.34361850004643,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":67.24770070798695,"decode_s":66.78022962401155,"prefill_s":0.4639597089262679,"decode_tps":61.33551835717616,"wall_s":72.32262604101561,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"extract","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":71.3470562499715,"decode_s":70.95234933297615,"prefill_s":0.3913552910089493,"decode_tps":57.72888478685403,"wall_s":78.3600482919719,"wall_kind":"fresh command","finish":"cap","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"qwen36","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2869,"phase_s":47.833778875065036,"decode_s":47.4565373751102,"prefill_s":0.37397554179187864,"decode_tps":60.45531677380071,"wall_s":54.22071829205379,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":60.45531677380071,"correct_answer_tps":60.45531677380071,"outcome":"pass","final_answer":"{\n \"A\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"B\": {\n \"eligible\": true,\n \"credit\": 8.5\n },\n \"C\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"D\": {\n \"eligible\": null,\n \"credit\": null\n },\n \"E\": {\n \"eligible\": true,\n \"credit\": null\n },\n \"F\": {\n \"eligible\": false,\n \"credit\": 0\n }\n}"}],"selected_latest_tasks":[{"model":"qwen36","task":"invoice","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":4913,"phase_s":78.16316241701134,"decode_s":77.72897954203654,"prefill_s":0.4289862079313025,"decode_tps":63.20679917511338,"wall_s":84.33732995891478,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":63.20679917511338,"correct_answer_tps":63.20679917511338,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"qwen36","task":"schedule","phase":"retry","budget":32768,"passed":true,"score":100.0,"content_passed":true,"tokens":18030,"phase_s":261.0228827910032,"decode_s":260.7873280410422,"prefill_s":0.2324907499132678,"decode_tps":69.13679485669823,"wall_s":268.3986142500071,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":69.13679485669823,"correct_answer_tps":69.13679485669823,"outcome":"pass","final_answer":"{\n \"makespan\": 11,\n \"starts\": {\n \"A\": 0,\n \"B\": 1,\n \"C\": 3,\n \"D\": 3,\n \"E\": 7,\n \"F\": 10\n }\n}"},{"model":"qwen36","task":"intervals","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":5530,"phase_s":81.24217783298809,"decode_s":81.04397283296566,"prefill_s":0.1953187909675762,"decode_tps":68.23456213576196,"wall_s":87.46063537499867,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":68.23456213576196,"correct_answer_tps":68.23456213576196,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for rec in records:\n u, s, e = rec['user'], rec['start'], rec['end']\n if e < s:\n raise ValueError(\"end < start\")\n user_intervals.setdefault(u, []).append((s, e))\n \n result = {}\n for u in sorted(user_intervals):\n intervals = user_intervals[u]\n intervals.sort()\n total = 0\n curr_s, curr_e = intervals[0]\n for s, e in intervals[1:]:\n if s >= curr_e:\n total += curr_e - curr_s\n curr_s, curr_e = s, e\n elif e > curr_e:\n curr_e = e\n total += curr_e - curr_s\n result[u] = total\n return result\n```"},{"model":"qwen36","task":"batches","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":7359,"phase_s":127.10947012493853,"decode_s":126.91229866701178,"prefill_s":0.1941449570003897,"decode_tps":57.98492405616493,"wall_s":132.67222404200584,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":57.98492405616493,"correct_answer_tps":57.98492405616493,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n if not graph:\n return []\n \n all_nodes = set(graph.keys())\n for prereqs in graph.values():\n all_nodes.update(prereqs)\n \n in_degree = {node: 0 for node in all_nodes}\n dependents = {node: [] for node in all_nodes}\n \n for node, prereqs in graph.items():\n unique_prereqs = set(prereqs)\n in_degree[node] = len(unique_prereqs)\n for p in unique_prereqs:\n dependents[p].append(node)\n \n queue = sorted([node for node in all_nodes if in_degree[node] == 0])\n \n batches = []\n processed_count = 0\n \n while queue:\n batch = sorted(queue)\n batches.append(batch)\n processed_count += len(batch)\n \n next_queue = []\n for node in batch:\n for dep in dependents[node]:\n in_degree[dep] -= 1\n if in_degree[dep] == 0:\n next_queue.append(dep)\n queue = next_queue\n \n if processed_count != len(all_nodes):\n raise ValueError(\"Cycle detected\")\n \n return batches\n```"},{"model":"qwen36","task":"sql","phase":"retry","budget":8192,"passed":false,"score":0.0,"content_passed":false,"tokens":5925,"phase_s":72.39941704203375,"decode_s":72.16814145899843,"prefill_s":0.22844279208220541,"decode_tps":82.09993884027381,"wall_s":78.32767183298711,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":82.09993884027381,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"```sql\nWITH order_gross AS (\n SELECT order_id, SUM(qty * unit_cents) AS gross\n FROM items\n GROUP BY order_id\n),\npaid_orders AS (\n SELECT o.id AS order_id, o.customer_id, COALESCE(og.gross, 0) AS gross\n FROM orders o\n LEFT JOIN order_gross og ON o.id = og.order_id\n WHERE o.status = 'paid'\n),\nsettled_refunds AS (\n SELECT order_id, SUM(amount_cents) AS refund_total\n FROM refunds\n WHERE status = 'settled'\n GROUP BY order_id\n),\norder_net AS (\n SELECT po.order_id, po.customer_id, po.gross - COALESCE(sr.refund_total, 0) AS net_cents\n FROM paid_orders po\n LEFT JOIN settled_refunds sr ON po.order_id = sr.order_id\n)\nSELECT\n c.id AS customer_id,\n COUNT(on.order_id) AS paid_order_count,\n COALESCE(SUM(on.net_cents), 0) AS net_cents\nFROM customers c\nLEFT JOIN order_net on ON c.id = on.customer_id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"qwen36","task":"cache","phase":"retry","budget":8192,"passed":false,"score":0,"content_passed":false,"tokens":6709,"phase_s":77.08092441700865,"decode_s":76.71267379191704,"prefill_s":0.365075750160031,"decode_tps":87.45621379588655,"wall_s":84.34613658301532,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":false,"accepted_pass":false,"fence_removed":true,"fence_grading":{"score":68.75,"passed":false,"checks":[{"name":"$ keys","passed":true},{"name":"$.read_results length","passed":false},{"name":"$.read_results[0]","passed":true,"expected":10,"actual":10},{"name":"$.read_results[1]","passed":true,"expected":null,"actual":null},{"name":"$.read_results[2]","passed":true,"expected":30,"actual":30},{"name":"$.read_results[3]","passed":true,"expected":null,"actual":null},{"name":"$.read_results[4]","passed":false,"expected":31,"actual":null},{"name":"$.read_results[5]","passed":true,"expected":null,"actual":null},{"name":"$.read_results[6]","passed":false,"expected":null,"actual":70},{"name":"$.read_results[7]","passed":false,"expected":70,"actual":""},{"name":"$.final_lru_to_mru length","passed":true},{"name":"$.final_lru_to_mru[0]","passed":true,"expected":"F","actual":"F"},{"name":"$.final_lru_to_mru[1]","passed":true,"expected":"H","actual":"H"},{"name":"$.final_lru_to_mru[2]","passed":true,"expected":"G","actual":"G"},{"name":"$.hits","passed":false,"expected":4,"actual":3},{"name":"$.misses","passed":true,"expected":4,"actual":4}]},"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":87.45621379588655,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"```json\n{\n \"read_results\": [10, null, 30, null, null, null, 70],\n \"final_lru_to_mru\": [\"F\", \"H\", \"G\"],\n \"hits\": 3,\n \"misses\": 4\n}\n```"},{"model":"qwen36","task":"extract","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":5611,"phase_s":72.84337199991569,"decode_s":72.52544087485876,"prefill_s":0.3151297509903088,"decode_tps":77.36595506784539,"wall_s":78.34604175004642,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":77.36595506784539,"correct_answer_tps":77.36595506784539,"outcome":"pass","final_answer":"[\n {\"id\": \"S-17\", \"status\": \"departed\", \"units\": 10, \"departed_utc\": \"2026-05-04T22:10:00Z\", \"evidence\": [3, 5]},\n {\"id\": \"S-18\", \"status\": \"planned\", \"units\": 9, \"departed_utc\": null, \"evidence\": [7]},\n {\"id\": \"S-19\", \"status\": \"departed\", \"units\": 5, \"departed_utc\": \"2026-05-05T02:50:00Z\", \"evidence\": [6]},\n {\"id\": \"S-20\", \"status\": \"cancelled\", \"units\": 0, \"departed_utc\": null, \"evidence\": [8]}\n]"},{"model":"qwen36","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2869,"phase_s":47.833778875065036,"decode_s":47.4565373751102,"prefill_s":0.37397554179187864,"decode_tps":60.45531677380071,"wall_s":54.22071829205379,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":null,"load_s":null,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":60.45531677380071,"correct_answer_tps":60.45531677380071,"outcome":"pass","final_answer":"{\n \"A\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"B\": {\n \"eligible\": true,\n \"credit\": 8.5\n },\n \"C\": {\n \"eligible\": false,\n \"credit\": 0\n },\n \"D\": {\n \"eligible\": null,\n \"credit\": null\n },\n \"E\": {\n \"eligible\": true,\n \"credit\": null\n },\n \"F\": {\n \"eligible\": false,\n \"credit\": 0\n }\n}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"spark","label":"Spark-X2.5 4B BF16","original":{"attempts":8,"accepted_passes":0,"strict_passes":0,"delivered":0,"incomplete":8,"native_recorded_tokens":32768,"partial_observed_tokens":0,"native_phase_s":1150.678895585239,"partial_generation_s":0,"weighted_native_decode_tps":28.532460594115225,"known_wall_s":1179.0135814577807,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":4,"strict_passes":4,"delivered":5,"incomplete":3,"native_recorded_tokens":104237,"partial_observed_tokens":0,"native_phase_s":3831.9022678750334,"partial_generation_s":0,"weighted_native_decode_tps":27.21486670840022,"known_wall_s":3857.840888083796,"missing_wall_count":0},"original_tasks":[{"model":"spark","task":"invoice","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":185.74219212506432,"decode_s":185.40576795802917,"prefill_s":0.33642416703514755,"decode_tps":22.092085079722132,"wall_s":189.81840820796788,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8695048526,"load_s":0.5973906250437722,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":177.03464216704015,"decode_s":176.8257096670568,"prefill_s":0.20893249998334798,"decode_tps":23.164052375145637,"wall_s":180.79208395804744,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8479649384,"load_s":0.5999937080778182,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"intervals","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":130.13552024995442,"decode_s":129.98120158398524,"prefill_s":0.15431866596918553,"decode_tps":31.512249079752014,"wall_s":132.47979283391032,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8479648100,"load_s":0.5269576250575483,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"batches","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":130.15520162507892,"decode_s":130.0009532920085,"prefill_s":0.15424833307042718,"decode_tps":31.507461262992077,"wall_s":132.68337454204448,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8479649168,"load_s":0.5299441249808297,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"sql","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":132.18938620889094,"decode_s":132.01573270896915,"prefill_s":0.1736534999217838,"decode_tps":31.026605056457164,"wall_s":136.15265637496486,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8479650408,"load_s":0.5951167088933289,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":132.15803229203448,"decode_s":131.37138512497768,"prefill_s":0.7866471670567989,"decode_tps":31.178783691009638,"wall_s":135.71008979098406,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8723045690,"load_s":0.5464583330322057,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"extract","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":131.59193187509663,"decode_s":131.37442787503824,"prefill_s":0.21750400005839765,"decode_tps":31.17806156230089,"wall_s":135.64065099996515,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8630517250,"load_s":0.5292943749809638,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"policy","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":131.67198904207908,"decode_s":131.47116387507413,"prefill_s":0.20082516700495034,"decode_tps":31.15512085898989,"wall_s":135.7365247498965,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8597046298,"load_s":0.5313231670297682,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""}],"selected_latest_tasks":[{"model":"spark","task":"invoice","phase":"retry","budget":16384,"passed":true,"score":100.0,"content_passed":true,"tokens":12926,"phase_s":458.863943375065,"decode_s":458.6203413750045,"prefill_s":0.24360200006049126,"decode_tps":28.18453268175184,"wall_s":461.76913133391645,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8894545486,"load_s":0.5457063750363886,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.18453268175184,"correct_answer_tps":28.18453268175184,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"spark","task":"schedule","phase":"retry","budget":32768,"passed":true,"score":100.0,"content_passed":true,"tokens":17198,"phase_s":608.69054379093,"decode_s":608.5183815408964,"prefill_s":0.17216225003357977,"decode_tps":28.262087919926184,"wall_s":612.3268846250139,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":9078276674,"load_s":0.5636450839228928,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.262087919926184,"correct_answer_tps":28.262087919926184,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"spark","task":"intervals","phase":"retry","budget":16384,"passed":false,"score":0,"content_passed":false,"tokens":16384,"phase_s":655.9193912071642,"decode_s":655.7381132910959,"prefill_s":0.1812779160682112,"decode_tps":24.98558443975453,"wall_s":659.3214209999423,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":9032955230,"load_s":0.7044257080415264,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"batches","phase":"retry","budget":16384,"passed":false,"score":0,"content_passed":false,"tokens":16384,"phase_s":608.4572567510186,"decode_s":608.2723776670173,"prefill_s":0.18487908400129527,"decode_tps":26.935301686457624,"wall_s":612.2484979159199,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":9002703426,"load_s":0.6368628749623895,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"spark","task":"sql","phase":"retry","budget":32768,"passed":true,"score":100.0,"content_passed":true,"tokens":21008,"phase_s":746.1294905839022,"decode_s":745.9553282499546,"prefill_s":0.17416233394760638,"decode_tps":28.162544329947654,"wall_s":748.6516744589899,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":9216612930,"load_s":0.7632806659676135,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":28.162544329947654,"correct_answer_tps":28.162544329947654,"outcome":"pass","final_answer":"```sql\nWITH paid_orders AS (\n SELECT o.id AS order_id, o.customer_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM orders o\n LEFT JOIN items i ON i.order_id = o.id\n WHERE o.status = 'paid'\n GROUP BY o.id, o.customer_id\n),\nrefunds_for_orders AS (\n SELECT r.order_id, SUM(r.amount_cents) AS total_refunds\n FROM refunds r\n JOIN orders o ON o.id = r.order_id\n WHERE r.status = 'settled' AND o.status = 'paid'\n GROUP BY r.order_id\n),\ncustomer_agg AS (\n SELECT \n c.id AS customer_id,\n COUNT(po.order_id) AS paid_order_count,\n COALESCE(SUM(po.gross), 0) - COALESCE(SUM(rf.total_refunds), 0) AS net_cents\n FROM customers c\n LEFT JOIN paid_orders po ON po.customer_id = c.id\n LEFT JOIN refunds_for_orders rf ON rf.order_id = po.order_id\n GROUP BY c.id\n)\nSELECT customer_id, paid_order_count, net_cents\nFROM customer_agg\nORDER BY customer_id ASC;\n```"},{"model":"spark","task":"cache","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":7519,"phase_s":252.31724266707897,"decode_s":252.00226908305194,"prefill_s":0.3149735840270296,"decode_tps":29.837032925770906,"wall_s":256.8834867499536,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8723045690,"load_s":0.8607372500700876,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":29.837032925770906,"correct_answer_tps":29.837032925770906,"outcome":"pass","final_answer":"{\n \"read_results\": [10, null, 30, null, 31, null, null, 70],\n \"final_lru_to_mru\": [\"F\", \"H\", \"G\"],\n \"hits\": 4,\n \"misses\": 4\n}"},{"model":"spark","task":"extract","phase":"retry","budget":8192,"passed":false,"score":80.0,"content_passed":false,"tokens":4626,"phase_s":174.8531681249151,"decode_s":174.61803620797582,"prefill_s":0.23513191693928093,"decode_tps":26.492108721749005,"wall_s":177.94992279203143,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8630517250,"load_s":0.5895034170243889,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":26.492108721749005,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\"3\", \"5\"]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\"7\"]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T01:50:00Z\",\n \"evidence\": [\"6\"]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\"8\"]\n }\n]"},{"model":"spark","task":"policy","phase":"retry","budget":8192,"passed":false,"score":0,"content_passed":false,"tokens":8192,"phase_s":326.67123137495946,"decode_s":326.4243543749908,"prefill_s":0.24687699996866286,"decode_tps":25.09616666221286,"wall_s":328.68986920802854,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8675661008,"load_s":0.6570633749943227,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"laguna","label":"Laguna-XS-2.1 8-bit","original":{"attempts":8,"accepted_passes":2,"strict_passes":2,"delivered":2,"incomplete":6,"native_recorded_tokens":29470,"partial_observed_tokens":0,"native_phase_s":501.36039629310835,"partial_generation_s":0,"weighted_native_decode_tps":59.72697382575969,"known_wall_s":557.766778747784,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":7,"strict_passes":7,"delivered":8,"incomplete":0,"native_recorded_tokens":37468,"partial_observed_tokens":0,"native_phase_s":677.10881770798,"partial_generation_s":0,"weighted_native_decode_tps":56.51478469992917,"known_wall_s":742.0627981668804,"missing_wall_count":0},"original_tasks":[{"model":"laguna","task":"invoice","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":63.07287954189815,"decode_s":62.44446479191539,"prefill_s":0.6284147499827668,"decode_tps":65.59428467597827,"wall_s":69.29817466600798,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":36959964424,"load_s":3.5093967500142753,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"laguna","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":64.89934591692872,"decode_s":63.866485666949295,"prefill_s":1.0328602499794215,"decode_tps":64.13379344778426,"wall_s":72.34542979102116,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":36959964424,"load_s":3.5558440829627216,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"laguna","task":"intervals","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":70.45498233404942,"decode_s":69.40416304196697,"prefill_s":1.0508192920824513,"decode_tps":59.016632727394914,"wall_s":78.37189900001977,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":36959964424,"load_s":3.7724790410138667,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"laguna","task":"batches","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":74.587738250033,"decode_s":73.3694888750324,"prefill_s":1.218249375000596,"decode_tps":55.82702105198754,"wall_s":81.47270304092672,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":36959964424,"load_s":3.7401274159783497,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"laguna","task":"sql","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":71.92310158303007,"decode_s":71.03827575000469,"prefill_s":0.8848258330253884,"decode_tps":57.659057131601756,"wall_s":78.38469699991401,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":36959964424,"load_s":3.5171386250294745,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"laguna","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":73.35336758405901,"decode_s":71.79625929205213,"prefill_s":1.5571082920068875,"decode_tps":57.05032602518094,"wall_s":81.41985879198182,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":36959964424,"load_s":3.867589208064601,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"laguna","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2908,"phase_s":49.97606854105834,"decode_s":48.74100245803129,"prefill_s":1.23506608302705,"decode_tps":59.66229362032407,"wall_s":57.273606374976225,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36959964424,"load_s":3.841083958046511,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":59.66229362032407,"correct_answer_tps":59.66229362032407,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\"L3\", \"L5\"]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\"L7\"]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [\"L6\"]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\"L8\"]\n }\n]"},{"model":"laguna","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1986,"phase_s":33.092912542051636,"decode_s":32.75176620809361,"prefill_s":0.3411463339580223,"decode_tps":60.6379511682402,"wall_s":39.20041008293629,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36959964424,"load_s":3.23660324991215,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":60.6379511682402,"correct_answer_tps":60.6379511682402,"outcome":"pass","final_answer":"{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}"}],"selected_latest_tasks":[{"model":"laguna","task":"invoice","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":4678,"phase_s":95.03192616591696,"decode_s":92.03908004099503,"prefill_s":2.9928461249219254,"decode_tps":50.82623596320581,"wall_s":105.56615883298218,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":41463679240,"load_s":7.33657833293546,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":50.82623596320581,"correct_answer_tps":50.82623596320581,"outcome":"pass","final_answer":"{\"billed\": {\"I1\": \"89.54\", \"I2\": \"70.55\", \"I3\": \"70.98\", \"I4\": \"191.45\"}, \"net_by_customer\": {\"Ada\": \"129.43\", \"Ben\": \"70.55\", \"Cy\": \"166.45\"}, \"grand_net\": \"366.43\"}"},{"model":"laguna","task":"schedule","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":7188,"phase_s":138.4117839578539,"decode_s":136.40548374992795,"prefill_s":2.0063002079259604,"decode_tps":52.695828660215405,"wall_s":147.88512962509412,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36673808136,"load_s":4.438114624936134,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":52.695828660215405,"correct_answer_tps":52.695828660215405,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"laguna","task":"intervals","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":4713,"phase_s":82.40748750104103,"decode_s":80.16880583402235,"prefill_s":2.2386816670186818,"decode_tps":58.78845207894913,"wall_s":90.47565104195382,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37530389768,"load_s":4.74924525001552,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":58.78845207894913,"correct_answer_tps":58.78845207894913,"outcome":"pass","final_answer":"def active_minutes(records):\n user_intervals = {}\n for record in records:\n user = record['user']\n start = record['start']\n end = record['end']\n if end < start:\n raise ValueError\n if user not in user_intervals:\n user_intervals[user] = []\n user_intervals[user].append((start, end))\n result = {}\n for user in sorted(user_intervals.keys()):\n intervals = user_intervals[user]\n sorted_intervals = sorted(intervals, key=lambda x: x[0])\n merged = [list(sorted_intervals[0])]\n for current in sorted_intervals[1:]:\n last = merged[-1]\n if current[0] <= last[1]:\n last[1] = max(last[1], current[1])\n else:\n merged.append(list(current))\n total = sum(interval[1] - interval[0] for interval in merged)\n result[user] = total\n return result"},{"model":"laguna","task":"batches","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":6686,"phase_s":116.29525941703469,"decode_s":114.42169879202265,"prefill_s":1.8735606250120327,"decode_tps":58.43297268425226,"wall_s":123.71755783399567,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36995065608,"load_s":4.456733207916841,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":58.43297268425226,"correct_answer_tps":58.43297268425226,"outcome":"pass","final_answer":"def dependency_batches(graph):\n all_nodes = set()\n for node in graph:\n all_nodes.add(node)\n for prereq in graph[node]:\n all_nodes.add(prereq)\n in_degree = {}\n for node in all_nodes:\n prereqs = graph.get(node, [])\n unique_prereqs = set(prereqs)\n in_degree[node] = len(unique_prereqs)\n dependents = {}\n for node in all_nodes:\n prereqs = graph.get(node, [])\n unique_prereqs = set(prereqs)\n for prereq in unique_prereqs:\n if prereq not in dependents:\n dependents[prereq] = []\n dependents[prereq].append(node)\n queue = [node for node in all_nodes if in_degree[node] == 0]\n batches = []\n processed = 0\n while queue:\n current_batch = sorted(queue)\n batches.append(current_batch)\n processed += len(current_batch)\n next_queue = []\n for node in current_batch:\n for dependent in dependents.get(node, []):\n in_degree[dependent] -= 1\n if in_degree[dependent] == 0:\n next_queue.append(dependent)\n queue = next_queue\n if processed != len(all_nodes):\n raise ValueError(\"Cycle detected\")\n return batches"},{"model":"laguna","task":"sql","phase":"retry","budget":8192,"passed":false,"score":0.0,"content_passed":false,"tokens":4353,"phase_s":73.05538008292206,"decode_s":71.32184849993791,"prefill_s":1.733531582984142,"decode_tps":61.033190972381895,"wall_s":81.42379979195539,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36959964424,"load_s":4.338668957934715,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":61.033190972381895,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"```sql\nSELECT c.id AS customer_id,\n COUNT(DISTINCT po.id) AS paid_order_count,\n COALESCE(SUM(po.gross), 0) - COALESCE(SUM(r.refund_amount), 0) AS net_cents\nFROM customers c\nLEFT JOIN (\n SELECT o.id AS order_id, o.customer_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM orders o\n LEFT JOIN items i ON o.id = i.order_id\n WHERE o.status = 'paid'\n GROUP BY o.id\n) po ON c.id = po.customer_id\nLEFT JOIN (\n SELECT r.order_id, SUM(r.amount_cents) AS refund_amount\n FROM refunds r\n WHERE r.status = 'settled'\n GROUP BY r.order_id\n) r ON po.order_id = r.order_id\nGROUP BY c.id\nORDER BY c.id;\n```"},{"model":"laguna","task":"cache","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":4956,"phase_s":88.83799950010143,"decode_s":87.12727166700643,"prefill_s":1.7107278330950066,"decode_tps":56.882304531943134,"wall_s":96.52048458298668,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36959964424,"load_s":4.392810499994084,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":56.882304531943134,"correct_answer_tps":56.882304531943134,"outcome":"pass","final_answer":"{\"read_results\": [10, null, 30, null, 31, null, null, 70], \"final_lru_to_mru\": [\"F\", \"H\", \"G\"], \"hits\": 4, \"misses\": 4}"},{"model":"laguna","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2908,"phase_s":49.97606854105834,"decode_s":48.74100245803129,"prefill_s":1.23506608302705,"decode_tps":59.66229362032407,"wall_s":57.273606374976225,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36959964424,"load_s":3.841083958046511,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":59.66229362032407,"correct_answer_tps":59.66229362032407,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [\"L3\", \"L5\"]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [\"L7\"]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [\"L6\"]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [\"L8\"]\n }\n]"},{"model":"laguna","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1986,"phase_s":33.092912542051636,"decode_s":32.75176620809361,"prefill_s":0.3411463339580223,"decode_tps":60.6379511682402,"wall_s":39.20041008293629,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":36959964424,"load_s":3.23660324991215,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":60.6379511682402,"correct_answer_tps":60.6379511682402,"outcome":"pass","final_answer":"{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"ornith","label":"Ornith-1.5 35B-A3B 8-bit","original":{"attempts":8,"accepted_passes":5,"strict_passes":4,"delivered":7,"incomplete":1,"native_recorded_tokens":22430,"partial_observed_tokens":0,"native_phase_s":360.3391206650995,"partial_generation_s":0,"weighted_native_decode_tps":64.60851970493988,"known_wall_s":431.36985904199537,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":5,"strict_passes":4,"delivered":8,"incomplete":0,"native_recorded_tokens":23524,"partial_observed_tokens":0,"native_phase_s":440.11956974794157,"partial_generation_s":0,"weighted_native_decode_tps":55.62812509944864,"known_wall_s":515.726516249124,"missing_wall_count":0},"original_tasks":[{"model":"ornith","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2333,"phase_s":37.288978833006695,"decode_s":35.70418637502007,"prefill_s":1.584792457986623,"decode_tps":65.34247764380511,"wall_s":48.24367808306124,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37433398720,"load_s":5.578088375041261,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.34247764380511,"correct_answer_tps":65.34247764380511,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"ornith","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":true,"tokens":3880,"phase_s":60.864373084157705,"decode_s":59.12649641709868,"prefill_s":1.7378766670590264,"decode_tps":65.62201779434288,"wall_s":69.35799616691656,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37164343160,"load_s":5.17521224997472,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.62201779434288,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"## Analysis\n\n**Critical path (lower bound on makespan):**\n- A → C → E → F = 3 + 4 + 3 + 1 = **11**\n- A → D → F = 3 + 2 + 1 = 6\n- B → E → F = 2 + 3 + 1 = 6 (but B ≥ 1)\n\nSo makespan ≥ 11.\n\n**Greedy lexicographic minimization** (minimize A, then B, then C, D, E, F):\n\n| Job | Constraint | Minimum |\n|-----|-----------|---------|\n| A | ≥ 0 | **0** |\n| B | ≥ 1 | **1** |\n| C | ≥ A+3 = 3 | **3** |\n| D | ≥ A+3 = 3, and D+2 ≤ 7 → D ≤ 5 | **3** |\n| E | ≥ max(B+2, C+4) = max(3, 7) | **7** |\n| F | ≥ max(D+2, E+3) = max(5, 10) | **10** |\n\n**Verify machine constraint (≤ 2 simultaneous):**\n- [0,1): A → 1\n- [1,3): A, B → 2 ✓\n- [3,5): C, D → 2 ✓\n- [5,7): C → 1\n- [7,10): E → 1\n- [10,11): F → 1\n\nAll intervals satisfy the two-machine limit. Makespan = F+1 = 11, matching the lower bound → optimal.\n\n## Answer\n\n```json\n{\"makespan\": 11, \"starts\": {\"A\": 0, \"B\": 1, \"C\": 3, \"D\": 3, \"E\": 7, \"F\": 10}}\n```"},{"model":"ornith","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2554,"phase_s":40.460254915989935,"decode_s":39.09869708295446,"prefill_s":1.3615578330354765,"decode_tps":65.32186979482358,"wall_s":48.30673754203599,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37081135592,"load_s":5.125834583071992,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.32186979482358,"correct_answer_tps":65.32186979482358,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n intervals = {}\n for rec in records:\n start = rec['start']\n end = rec['end']\n if end < start:\n raise ValueError('end < start')\n intervals.setdefault(rec['user'], []).append((start, end))\n result = {}\n for user in sorted(intervals):\n ivs = sorted(intervals[user])\n total = 0\n cur_s, cur_e = ivs[0]\n for s, e in ivs[1:]:\n if s > cur_e:\n total += cur_e - cur_s\n cur_s, cur_e = s, e\n elif e > cur_e:\n cur_e = e\n total += cur_e - cur_s\n result[user] = total\n return result\n```"},{"model":"ornith","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2923,"phase_s":46.39529583300463,"decode_s":44.576435207971365,"prefill_s":1.8188606250332668,"decode_tps":65.57276252268139,"wall_s":54.27849195804447,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37093307720,"load_s":5.154447791981511,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.57276252268139,"correct_answer_tps":65.57276252268139,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n prereqs = {}\n for node, prereq_list in graph.items():\n prereqs[node] = set(prereq_list)\n for node, prereq_list in graph.items():\n for p in prereq_list:\n if p not in prereqs:\n prereqs[p] = set()\n if not prereqs:\n return []\n batches = []\n remaining = set(prereqs)\n done = set()\n while remaining:\n available = sorted(n for n in remaining if prereqs[n] <= done)\n if not available:\n raise ValueError(\"cycle detected\")\n batches.append(available)\n done.update(available)\n remaining.difference_update(available)\n return batches\n```"},{"model":"ornith","task":"sql","phase":"original","budget":4096,"passed":false,"score":0.0,"content_passed":false,"tokens":2707,"phase_s":43.81692112388555,"decode_s":42.2408056659624,"prefill_s":1.5761154579231518,"decode_tps":64.08495191608758,"wall_s":51.30265495798085,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37147642968,"load_s":4.994120749994181,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":64.08495191608758,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"```sql\nWITH paid_orders AS (\n SELECT o.id AS order_id, o.customer_id AS customer_id\n FROM orders o\n WHERE o.status = 'paid'\n),\norder_gross AS (\n SELECT po.order_id, po.customer_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.order_id\n GROUP BY po.order_id, po.customer_id\n),\norder_refunds AS (\n SELECT po.order_id, po.customer_id,\n COALESCE(SUM(r.amount_cents), 0) AS refunds\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.order_id AND r.status = 'settled'\n GROUP BY po.order_id, po.customer_id\n),\norder_net AS (\n SELECT og.order_id, og.customer_id,\n og.gross - or_.refunds AS net\n FROM order_gross og\n JOIN order_refunds or_ ON or_.order_id = og.order_id\n)\nSELECT c.id AS customer_id,\n COUNT(on.order_id) AS paid_order_count,\n COALESCE(SUM(on.net), 0) AS net_cents\nFROM customers c\nLEFT JOIN order_net on_ ON on_.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"ornith","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":67.04516595904715,"decode_s":65.28305299999192,"prefill_s":1.7621129590552302,"decode_tps":62.742163728165515,"wall_s":75.41336883394979,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":37499968068,"load_s":5.136967458063737,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ornith","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2912,"phase_s":47.178375250077806,"decode_s":45.50510991702322,"prefill_s":1.6732653330545872,"decode_tps":63.99281323152318,"wall_s":57.29671470797621,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37367748752,"load_s":5.06543800001964,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":63.99281323152318,"correct_answer_tps":63.99281323152318,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"ornith","task":"policy","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":true,"tokens":1025,"phase_s":17.289755665929988,"decode_s":15.63305274990853,"prefill_s":1.6567029160214588,"decode_tps":65.5662087499831,"wall_s":27.17021679203026,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37336491008,"load_s":5.028456500032917,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.A keys","passed":true},{"name":"$.A.eligible","passed":true,"expected":false,"actual":false},{"name":"$.A.credit","passed":true,"expected":0,"actual":0},{"name":"$.B keys","passed":true},{"name":"$.B.eligible","passed":true,"expected":true,"actual":true},{"name":"$.B.credit","passed":true,"expected":8.5,"actual":8.5},{"name":"$.C keys","passed":true},{"name":"$.C.eligible","passed":true,"expected":false,"actual":false},{"name":"$.C.credit","passed":true,"expected":0,"actual":0},{"name":"$.D keys","passed":true},{"name":"$.D.eligible","passed":true,"expected":null,"actual":null},{"name":"$.D.credit","passed":true,"expected":null,"actual":null},{"name":"$.E keys","passed":true},{"name":"$.E.eligible","passed":true,"expected":true,"actual":true},{"name":"$.E.credit","passed":true,"expected":null,"actual":null},{"name":"$.F keys","passed":true},{"name":"$.F.eligible","passed":true,"expected":false,"actual":false},{"name":"$.F.credit","passed":true,"expected":0,"actual":0}]},"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.5662087499831,"correct_answer_tps":65.5662087499831,"outcome":"pass","final_answer":"```json\n{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}\n```"}],"selected_latest_tasks":[{"model":"ornith","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2333,"phase_s":37.288978833006695,"decode_s":35.70418637502007,"prefill_s":1.584792457986623,"decode_tps":65.34247764380511,"wall_s":48.24367808306124,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37433398720,"load_s":5.578088375041261,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.34247764380511,"correct_answer_tps":65.34247764380511,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"ornith","task":"schedule","phase":"retry","budget":8192,"passed":false,"score":0,"content_passed":true,"tokens":3880,"phase_s":80.01768754096702,"decode_s":76.54678229091223,"prefill_s":3.470905250054784,"decode_tps":50.68795687915729,"wall_s":90.4345729999477,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37164343160,"load_s":6.541910500032827,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":50.68795687915729,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"## Analysis\n\n**Critical path (lower bound on makespan):**\n- A → C → E → F = 3 + 4 + 3 + 1 = **11**\n- A → D → F = 3 + 2 + 1 = 6\n- B → E → F = 2 + 3 + 1 = 6 (but B ≥ 1)\n\nSo makespan ≥ 11.\n\n**Greedy lexicographic minimization** (minimize A, then B, then C, D, E, F):\n\n| Job | Constraint | Minimum |\n|-----|-----------|---------|\n| A | ≥ 0 | **0** |\n| B | ≥ 1 | **1** |\n| C | ≥ A+3 = 3 | **3** |\n| D | ≥ A+3 = 3, and D+2 ≤ 7 → D ≤ 5 | **3** |\n| E | ≥ max(B+2, C+4) = max(3, 7) | **7** |\n| F | ≥ max(D+2, E+3) = max(5, 10) | **10** |\n\n**Verify machine constraint (≤ 2 simultaneous):**\n- [0,1): A → 1\n- [1,3): A, B → 2 ✓\n- [3,5): C, D → 2 ✓\n- [5,7): C → 1\n- [7,10): E → 1\n- [10,11): F → 1\n\nAll intervals satisfy the two-machine limit. Makespan = F+1 = 11, matching the lower bound → optimal.\n\n## Answer\n\n```json\n{\"makespan\": 11, \"starts\": {\"A\": 0, \"B\": 1, \"C\": 3, \"D\": 3, \"E\": 7, \"F\": 10}}\n```"},{"model":"ornith","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2554,"phase_s":40.460254915989935,"decode_s":39.09869708295446,"prefill_s":1.3615578330354765,"decode_tps":65.32186979482358,"wall_s":48.30673754203599,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37081135592,"load_s":5.125834583071992,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.32186979482358,"correct_answer_tps":65.32186979482358,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n intervals = {}\n for rec in records:\n start = rec['start']\n end = rec['end']\n if end < start:\n raise ValueError('end < start')\n intervals.setdefault(rec['user'], []).append((start, end))\n result = {}\n for user in sorted(intervals):\n ivs = sorted(intervals[user])\n total = 0\n cur_s, cur_e = ivs[0]\n for s, e in ivs[1:]:\n if s > cur_e:\n total += cur_e - cur_s\n cur_s, cur_e = s, e\n elif e > cur_e:\n cur_e = e\n total += cur_e - cur_s\n result[user] = total\n return result\n```"},{"model":"ornith","task":"batches","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2923,"phase_s":46.39529583300463,"decode_s":44.576435207971365,"prefill_s":1.8188606250332668,"decode_tps":65.57276252268139,"wall_s":54.27849195804447,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37093307720,"load_s":5.154447791981511,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":65.57276252268139,"correct_answer_tps":65.57276252268139,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n prereqs = {}\n for node, prereq_list in graph.items():\n prereqs[node] = set(prereq_list)\n for node, prereq_list in graph.items():\n for p in prereq_list:\n if p not in prereqs:\n prereqs[p] = set()\n if not prereqs:\n return []\n batches = []\n remaining = set(prereqs)\n done = set()\n while remaining:\n available = sorted(n for n in remaining if prereqs[n] <= done)\n if not available:\n raise ValueError(\"cycle detected\")\n batches.append(available)\n done.update(available)\n remaining.difference_update(available)\n return batches\n```"},{"model":"ornith","task":"sql","phase":"retry","budget":8192,"passed":false,"score":0.0,"content_passed":false,"tokens":2707,"phase_s":56.02597558195703,"decode_s":53.73733579099644,"prefill_s":2.2886397909605876,"decode_tps":50.37465963196395,"wall_s":66.31524745805655,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37147642968,"load_s":5.662449417053722,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":50.37465963196395,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"```sql\nWITH paid_orders AS (\n SELECT o.id AS order_id, o.customer_id AS customer_id\n FROM orders o\n WHERE o.status = 'paid'\n),\norder_gross AS (\n SELECT po.order_id, po.customer_id,\n COALESCE(SUM(i.qty * i.unit_cents), 0) AS gross\n FROM paid_orders po\n LEFT JOIN items i ON i.order_id = po.order_id\n GROUP BY po.order_id, po.customer_id\n),\norder_refunds AS (\n SELECT po.order_id, po.customer_id,\n COALESCE(SUM(r.amount_cents), 0) AS refunds\n FROM paid_orders po\n LEFT JOIN refunds r ON r.order_id = po.order_id AND r.status = 'settled'\n GROUP BY po.order_id, po.customer_id\n),\norder_net AS (\n SELECT og.order_id, og.customer_id,\n og.gross - or_.refunds AS net\n FROM order_gross og\n JOIN order_refunds or_ ON or_.order_id = og.order_id\n)\nSELECT c.id AS customer_id,\n COUNT(on.order_id) AS paid_order_count,\n COALESCE(SUM(on.net), 0) AS net_cents\nFROM customers c\nLEFT JOIN order_net on_ ON on_.customer_id = c.id\nGROUP BY c.id\nORDER BY c.id ASC;\n```"},{"model":"ornith","task":"cache","phase":"retry","budget":8192,"passed":false,"score":0,"content_passed":true,"tokens":5190,"phase_s":110.52495466696564,"decode_s":107.90182516700588,"prefill_s":2.6231294999597594,"decode_tps":48.09927906193558,"wall_s":120.66662274999544,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37499968068,"load_s":5.398723250022158,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":48.09927906193558,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"# LRU Cache Simulation (capacity 3, TTL=4)\n\nTracing through events (removing expired entries with expiry ≤ t before each):\n\n| t | op | Pre-clean (expiry≤t) | Action | State (LRU→MRU) | read |\n|---|-----|------|------|------|------|\n| 0 | put A=10 | — | A(10,exp4) | A | — |\n| 1 | put B=20 | — | B(20,exp5) | A,B | — |\n| 2 | get A | — | hit, A→MRU | B,A | **10** |\n| 3 | put C=30 | — | C(30,exp7) | B,A,C | — |\n| 3 | put D=40 | — | new→evict B, D(40,exp7) | A,C,D | — |\n| 4 | get A | remove A(exp4) | miss | C,D | **null** |\n| 4 | get C | — | hit, C→MRU | D,C | **30** |\n| 5 | put C=31 | — | overwrite, C(31,exp9) | D,C | — |\n| 6 | put E=50 | — | E(50,exp10) | D,C,E | — |\n| 7 | get D | remove D(exp7) | miss | C,E | **null** |\n| 7 | put F=60 | — | F(60,exp11) | C,E,F | — |\n| 8 | get C | — | hit, C→MRU | E,F,C | **31** |\n| 9 | get C | remove C(exp9) | miss | E,F | **null** |\n| 9 | put G=70 | — | G(70,exp13) | E,F,G | — |\n| 10 | get E | remove E(exp10) | miss | F,G | **null** |\n| 10 | put F=61 | — | overwrite, F(61,exp14) | G,F | — |\n| 11 | put H=80 | — | H(80,exp15) | G,F,H | — |\n| 12 | get G | — | hit, G→MRU | F,H,G | **70** |\n\n```json\n{\n \"read_results\": [10, null, 30, null, 31, null, null, 70],\n \"final_lru_to_mru\": [\"F\", \"H\", \"G\"],\n \"hits\": 4,\n \"misses\": 4\n}\n```"},{"model":"ornith","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2912,"phase_s":47.178375250077806,"decode_s":45.50510991702322,"prefill_s":1.6732653330545872,"decode_tps":63.99281323152318,"wall_s":57.29671470797621,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37367748752,"load_s":5.06543800001964,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":63.99281323152318,"correct_answer_tps":63.99281323152318,"outcome":"pass","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"ornith","task":"policy","phase":"retry","budget":8192,"passed":false,"score":0,"content_passed":true,"tokens":1025,"phase_s":22.228047125972807,"decode_s":19.809240916976705,"prefill_s":2.418806208996102,"decode_tps":51.74352739188332,"wall_s":30.184450750006363,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":37336491008,"load_s":5.423407125053927,"strict_pass":false,"accepted_pass":true,"fence_removed":true,"fence_grading":{"score":100.0,"passed":true,"checks":[{"name":"$ keys","passed":true},{"name":"$.A keys","passed":true},{"name":"$.A.eligible","passed":true,"expected":false,"actual":false},{"name":"$.A.credit","passed":true,"expected":0,"actual":0},{"name":"$.B keys","passed":true},{"name":"$.B.eligible","passed":true,"expected":true,"actual":true},{"name":"$.B.credit","passed":true,"expected":8.5,"actual":8.5},{"name":"$.C keys","passed":true},{"name":"$.C.eligible","passed":true,"expected":false,"actual":false},{"name":"$.C.credit","passed":true,"expected":0,"actual":0},{"name":"$.D keys","passed":true},{"name":"$.D.eligible","passed":true,"expected":null,"actual":null},{"name":"$.D.credit","passed":true,"expected":null,"actual":null},{"name":"$.E keys","passed":true},{"name":"$.E.eligible","passed":true,"expected":true,"actual":true},{"name":"$.E.credit","passed":true,"expected":null,"actual":null},{"name":"$.F keys","passed":true},{"name":"$.F.eligible","passed":true,"expected":false,"actual":false},{"name":"$.F.credit","passed":true,"expected":0,"actual":0}]},"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":51.74352739188332,"correct_answer_tps":51.74352739188332,"outcome":"pass","final_answer":"```json\n{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}\n```"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"ternary","label":"Ternary Bonsai 27B 2-bit","original":{"attempts":8,"accepted_passes":2,"strict_passes":2,"delivered":2,"incomplete":6,"native_recorded_tokens":31484,"partial_observed_tokens":0,"native_phase_s":1261.64564941579,"partial_generation_s":0,"weighted_native_decode_tps":25.077057178073414,"known_wall_s":1287.0737197498092,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":6,"strict_passes":6,"delivered":7,"incomplete":1,"native_recorded_tokens":61965,"partial_observed_tokens":0,"native_phase_s":3080.091288918047,"partial_generation_s":0,"weighted_native_decode_tps":20.16216726768287,"known_wall_s":3106.453283249866,"missing_wall_count":0},"original_tasks":[{"model":"ternary","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":4015,"phase_s":160.28699129191227,"decode_s":159.25944187492132,"prefill_s":1.0275494169909507,"decode_tps":25.210436208568957,"wall_s":162.7353508749511,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8836141530,"load_s":1.061744875041768,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":25.210436208568957,"correct_answer_tps":25.210436208568957,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"ternary","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":162.55582141596824,"decode_s":161.96404412505217,"prefill_s":0.5917772909160703,"decode_tps":25.289563632021224,"wall_s":165.78146370896138,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8492715568,"load_s":1.0842056670226157,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"intervals","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":160.93121849989984,"decode_s":160.4517857079627,"prefill_s":0.4794327919371426,"decode_tps":25.527917822336388,"wall_s":162.76998504204676,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8325364212,"load_s":1.069882040959783,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"batches","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":161.26554604107514,"decode_s":160.7993928330252,"prefill_s":0.46615320804994553,"decode_tps":25.472732998770116,"wall_s":165.76268879091367,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8307689516,"load_s":1.1247187500121072,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"sql","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":161.588614749955,"decode_s":161.00330629199743,"prefill_s":0.5853084579575807,"decode_tps":25.440471343932824,"wall_s":165.81893574993592,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8445447448,"load_s":1.021858291933313,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":163.48218433302827,"decode_s":162.23515725007746,"prefill_s":1.247027082950808,"decode_tps":25.247301937681847,"wall_s":165.81993579107802,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8974898480,"load_s":1.0589902499923483,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"extract","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":164.32668016699608,"decode_s":163.3905645830091,"prefill_s":0.9361155839869754,"decode_tps":25.068767039598935,"wall_s":168.81173591699917,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":8786746278,"load_s":1.0865666669560596,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2893,"phase_s":127.20859291695524,"decode_s":126.3865259999875,"prefill_s":0.8220669169677421,"decode_tps":22.890098268863614,"wall_s":129.57362387492321,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8720832906,"load_s":1.1036211249884218,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":22.890098268863614,"correct_answer_tps":22.890098268863614,"outcome":"pass","final_answer":"{\"A\": {\"eligible\": false, \"credit\": 0}, \"B\": {\"eligible\": true, \"credit\": 8.5}, \"C\": {\"eligible\": false, \"credit\": 0}, \"D\": {\"eligible\": null, \"credit\": null}, \"E\": {\"eligible\": true, \"credit\": null}, \"F\": {\"eligible\": false, \"credit\": 0}}"}],"selected_latest_tasks":[{"model":"ternary","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":4015,"phase_s":160.28699129191227,"decode_s":159.25944187492132,"prefill_s":1.0275494169909507,"decode_tps":25.210436208568957,"wall_s":162.7353508749511,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8836141530,"load_s":1.061744875041768,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":25.210436208568957,"correct_answer_tps":25.210436208568957,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"ternary","task":"schedule","phase":"retry","budget":16384,"passed":true,"score":100.0,"content_passed":true,"tokens":15071,"phase_s":801.8546358749736,"decode_s":801.1655271670315,"prefill_s":0.6891087079420686,"decode_tps":18.811343585004142,"wall_s":804.8935082500102,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8994434466,"load_s":1.1543897500960156,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":18.811343585004142,"correct_answer_tps":18.811343585004142,"outcome":"pass","final_answer":"{\"makespan\": 11, \"starts\": {\"A\": 0, \"B\": 1, \"C\": 3, \"D\": 3, \"E\": 7, \"F\": 10}}"},{"model":"ternary","task":"intervals","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":5236,"phase_s":273.4445227079559,"decode_s":272.8992529580137,"prefill_s":0.5452697499422356,"decode_tps":19.18656772873458,"wall_s":277.1489260420203,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8325364212,"load_s":1.2142826670315117,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":19.18656772873458,"correct_answer_tps":19.18656772873458,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_ivs = {}\n for r in records:\n u, s, e = r['user'], r['start'], r['end']\n if e < s:\n raise ValueError(\"end < start\")\n user_ivs.setdefault(u, []).append((s, e))\n \n result = {}\n for u in sorted(user_ivs):\n user_ivs[u].sort(key=lambda x: x[0])\n merged = []\n for s, e in user_ivs[u]:\n if not merged or s >= merged[-1][1]:\n merged.append([s, e])\n elif e > merged[-1][1]:\n merged[-1][1] = e\n result[u] = sum(e - s for s, e in merged)\n return result\n```"},{"model":"ternary","task":"batches","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":6994,"phase_s":365.04155166703276,"decode_s":364.49233729206026,"prefill_s":0.5492143749725074,"decode_tps":19.188332056472976,"wall_s":367.5240319579607,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8357072842,"load_s":1.194826832972467,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":19.188332056472976,"correct_answer_tps":19.188332056472976,"outcome":"pass","final_answer":"```python\ndef dependency_batches(graph):\n all_nodes = set(graph.keys())\n for reqs in graph.values():\n all_nodes.update(reqs)\n\n prereqs = {node: set() for node in all_nodes}\n dependents = {node: set() for node in all_nodes}\n\n for node, reqs in graph.items():\n prereqs[node] = set(reqs)\n for req in reqs:\n dependents[req].add(node)\n\n in_degree = {node: len(prereqs[node]) for node in all_nodes}\n current_batch = [node for node in all_nodes if in_degree[node] == 0]\n processed = len(current_batch)\n batches = []\n\n while current_batch:\n current_batch.sort()\n batches.append(current_batch)\n next_batch = []\n for node in current_batch:\n for dep in dependents[node]:\n in_degree[dep] -= 1\n if in_degree[dep] == 0:\n next_batch.append(dep)\n processed += 1\n current_batch = next_batch\n\n if processed < len(all_nodes):\n raise ValueError(\"Cycle detected\")\n\n return batches\n```"},{"model":"ternary","task":"sql","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":6725,"phase_s":355.13360170810483,"decode_s":354.44783920806367,"prefill_s":0.6857625000411645,"decode_tps":18.97317251256361,"wall_s":358.74280404206365,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8445447448,"load_s":1.4277037919964641,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":18.97317251256361,"correct_answer_tps":18.97317251256361,"outcome":"pass","final_answer":"```sql\nWITH order_gross AS (\n SELECT order_id, COALESCE(SUM(qty * unit_cents), 0) AS gross\n FROM items\n GROUP BY order_id\n),\norder_refunds AS (\n SELECT order_id, COALESCE(SUM(amount_cents), 0) AS refunds\n FROM refunds\n WHERE status = 'settled'\n GROUP BY order_id\n)\nSELECT\n c.id AS customer_id,\n COALESCE(p.order_count, 0) AS paid_order_count,\n COALESCE(p.net_cents, 0) AS net_cents\nFROM customers c\nLEFT JOIN (\n SELECT\n o.customer_id,\n COUNT(o.id) AS order_count,\n SUM(COALESCE(og.gross, 0) - COALESCE(orf.refunds, 0)) AS net_cents\n FROM orders o\n LEFT JOIN order_gross og ON o.id = og.order_id\n LEFT JOIN order_refunds orf ON o.id = orf.order_id\n WHERE o.status = 'paid'\n GROUP BY o.customer_id\n) p ON c.id = p.customer_id\nORDER BY c.id ASC;\n```"},{"model":"ternary","task":"cache","phase":"retry","budget":16384,"passed":false,"score":0,"content_passed":false,"tokens":16384,"phase_s":841.2211266260128,"decode_s":839.6964987920364,"prefill_s":1.5246278339764103,"decode_tps":19.51181173622798,"wall_s":846.1479033749783,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":9117145810,"load_s":1.216815958963707,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"ternary","task":"extract","phase":"retry","budget":8192,"passed":false,"score":93.33333333333333,"content_passed":false,"tokens":4647,"phase_s":155.90026612509973,"decode_s":154.98289766605012,"prefill_s":0.9173684590496122,"decode_tps":29.983953519911193,"wall_s":159.6871348329587,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8786746278,"load_s":0.9930654580239207,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":29.983953519911193,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"[\n {\n \"id\": \"S-17\",\n \"status\": \"departed\",\n \"units\": 10,\n \"departed_utc\": \"2026-05-04T22:10:00Z\",\n \"evidence\": [3, 5]\n },\n {\n \"id\": \"S-18\",\n \"status\": \"planned\",\n \"units\": 9,\n \"departed_utc\": null,\n \"evidence\": [4, 7]\n },\n {\n \"id\": \"S-19\",\n \"status\": \"departed\",\n \"units\": 5,\n \"departed_utc\": \"2026-05-05T02:50:00Z\",\n \"evidence\": [6]\n },\n {\n \"id\": \"S-20\",\n \"status\": \"cancelled\",\n \"units\": 0,\n \"departed_utc\": null,\n \"evidence\": [8]\n }\n]"},{"model":"ternary","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":2893,"phase_s":127.20859291695524,"decode_s":126.3865259999875,"prefill_s":0.8220669169677421,"decode_tps":22.890098268863614,"wall_s":129.57362387492321,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":8720832906,"load_s":1.1036211249884218,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":22.890098268863614,"correct_answer_tps":22.890098268863614,"outcome":"pass","final_answer":"{\"A\": {\"eligible\": false, \"credit\": 0}, \"B\": {\"eligible\": true, \"credit\": 8.5}, \"C\": {\"eligible\": false, \"credit\": 0}, \"D\": {\"eligible\": null, \"credit\": null}, \"E\": {\"eligible\": true, \"credit\": null}, \"F\": {\"eligible\": false, \"credit\": 0}}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."},{"id":"minicpm","label":"MiniCPM5-2B BF16","original":{"attempts":8,"accepted_passes":4,"strict_passes":4,"delivered":4,"incomplete":4,"native_recorded_tokens":26172,"partial_observed_tokens":0,"native_phase_s":561.5618911653291,"partial_generation_s":0,"weighted_native_decode_tps":46.71330001745904,"known_wall_s":584.7852681251243,"missing_wall_count":0},"selected_latest":{"attempts":8,"accepted_passes":6,"strict_passes":6,"delivered":7,"incomplete":1,"native_recorded_tokens":32707,"partial_observed_tokens":0,"native_phase_s":709.9262469579699,"partial_generation_s":0,"weighted_native_decode_tps":46.13906554886659,"known_wall_s":732.5174733309541,"missing_wall_count":0},"original_tasks":[{"model":"minicpm","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3323,"phase_s":64.96895287407096,"decode_s":64.84655508305877,"prefill_s":0.12239779101219028,"decode_tps":51.24404828820486,"wall_s":66.2833240829641,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5415753552,"load_s":0.472462791018188,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":51.24404828820486,"correct_answer_tps":51.24404828820486,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"minicpm","task":"schedule","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":89.1436480000848,"decode_s":89.023623917019,"prefill_s":0.12002408306580037,"decode_tps":46.01025907255784,"wall_s":93.42522591701709,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":5276738568,"load_s":0.6170674579916522,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"minicpm","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1904,"phase_s":40.40815991710406,"decode_s":40.30203704209998,"prefill_s":0.10612287500407547,"decode_tps":47.24326956503611,"wall_s":42.17938758304808,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5178708748,"load_s":0.5095744580030441,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":47.24326956503611,"correct_answer_tps":47.24326956503611,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for rec in records:\n user = rec['user']\n start = rec['start']\n end = rec['end']\n if end < start:\n raise ValueError(\"end must be >= start\")\n user_intervals.setdefault(user, []).append((start, end))\n\n result = {}\n for user, intervals in user_intervals.items():\n intervals.sort(key=lambda x: x[0])\n merged = []\n for start, end in intervals:\n if not merged:\n merged.append([start, end])\n else:\n last_start, last_end = merged[-1]\n if start < last_end:\n merged[-1][1] = max(last_end, end)\n else:\n merged.append([start, end])\n total = sum(end - start for start, end in merged)\n result[user] = total\n\n return {user: result[user] for user in sorted(result.keys())}\n```"},{"model":"minicpm","task":"batches","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":86.63353733404074,"decode_s":86.53730850003194,"prefill_s":0.096228834008798,"decode_tps":47.33218620958714,"wall_s":90.43135904206429,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":5276738336,"load_s":0.5659717499511316,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"minicpm","task":"sql","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":90.44983024988323,"decode_s":90.32610849989578,"prefill_s":0.12372174998745322,"decode_tps":45.34680025548456,"wall_s":93.42967258300632,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":5276738588,"load_s":0.5764448340050876,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"minicpm","task":"cache","phase":"original","budget":4096,"passed":false,"score":0,"content_passed":false,"tokens":4096,"phase_s":90.62081524904352,"decode_s":90.18122879101429,"prefill_s":0.43958645802922547,"decode_tps":45.419651682636285,"wall_s":93.52252779202536,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":5477325032,"load_s":0.5751085840165615,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"minicpm","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3010,"phase_s":65.45775062509347,"decode_s":65.31304254208226,"prefill_s":0.14470808301120996,"decode_tps":46.08574157390705,"wall_s":69.34674516704399,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5352052252,"load_s":0.6402295840671286,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":46.08574157390705,"correct_answer_tps":46.08574157390705,"outcome":"pass","final_answer":"[\n {\"id\": \"S-17\", \"status\": \"departed\", \"units\": 10, \"departed_utc\": \"2026-05-04T22:10:00Z\", \"evidence\": [\"L3\", \"L5\"]},\n {\"id\": \"S-18\", \"status\": \"planned\", \"units\": 9, \"departed_utc\": null, \"evidence\": [\"L7\"]},\n {\"id\": \"S-19\", \"status\": \"departed\", \"units\": 5, \"departed_utc\": \"2026-05-05T02:50:00Z\", \"evidence\": [\"L6\"]},\n {\"id\": \"S-20\", \"status\": \"cancelled\", \"units\": 0, \"departed_utc\": null, \"evidence\": [\"L8\"]}\n]"},{"model":"minicpm","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1551,"phase_s":33.87919691600837,"decode_s":33.7387983329827,"prefill_s":0.14039858302567154,"decode_tps":45.970813325730056,"wall_s":36.16702595795505,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5342041584,"load_s":0.5971827920293435,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":45.970813325730056,"correct_answer_tps":45.970813325730056,"outcome":"pass","final_answer":"{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}"}],"selected_latest_tasks":[{"model":"minicpm","task":"invoice","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3323,"phase_s":64.96895287407096,"decode_s":64.84655508305877,"prefill_s":0.12239779101219028,"decode_tps":51.24404828820486,"wall_s":66.2833240829641,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5415753552,"load_s":0.472462791018188,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":51.24404828820486,"correct_answer_tps":51.24404828820486,"outcome":"pass","final_answer":"{\n \"billed\": {\n \"I1\": \"89.54\",\n \"I2\": \"70.55\",\n \"I3\": \"70.98\",\n \"I4\": \"191.45\"\n },\n \"net_by_customer\": {\n \"Ada\": \"129.43\",\n \"Ben\": \"70.55\",\n \"Cy\": \"166.45\"\n },\n \"grand_net\": \"366.43\"\n}"},{"model":"minicpm","task":"schedule","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":4201,"phase_s":82.54560066689737,"decode_s":82.42796795791946,"prefill_s":0.1176327089779079,"decode_tps":50.965711081761285,"wall_s":84.36040179093834,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5290632200,"load_s":0.8266849999781698,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":50.965711081761285,"correct_answer_tps":50.965711081761285,"outcome":"pass","final_answer":"{\"makespan\":11,\"starts\":{\"A\":0,\"B\":1,\"C\":3,\"D\":3,\"E\":7,\"F\":10}}"},{"model":"minicpm","task":"intervals","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1904,"phase_s":40.40815991710406,"decode_s":40.30203704209998,"prefill_s":0.10612287500407547,"decode_tps":47.24326956503611,"wall_s":42.17938758304808,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5178708748,"load_s":0.5095744580030441,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":47.24326956503611,"correct_answer_tps":47.24326956503611,"outcome":"pass","final_answer":"```python\ndef active_minutes(records):\n user_intervals = {}\n for rec in records:\n user = rec['user']\n start = rec['start']\n end = rec['end']\n if end < start:\n raise ValueError(\"end must be >= start\")\n user_intervals.setdefault(user, []).append((start, end))\n\n result = {}\n for user, intervals in user_intervals.items():\n intervals.sort(key=lambda x: x[0])\n merged = []\n for start, end in intervals:\n if not merged:\n merged.append([start, end])\n else:\n last_start, last_end = merged[-1]\n if start < last_end:\n merged[-1][1] = max(last_end, end)\n else:\n merged.append([start, end])\n total = sum(end - start for start, end in merged)\n result[user] = total\n\n return {user: result[user] for user in sorted(result.keys())}\n```"},{"model":"minicpm","task":"batches","phase":"retry","budget":8192,"passed":false,"score":20.0,"content_passed":false,"tokens":4977,"phase_s":107.88602962589357,"decode_s":107.78200929192826,"prefill_s":0.10402033396530896,"decode_tps":46.1765375566507,"wall_s":111.60205316601787,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5332312868,"load_s":0.731478541973047,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":46.1765375566507,"correct_answer_tps":0.0,"outcome":"wrong","final_answer":"```python\ndef dependency_batches(graph):\n if not graph:\n return []\n\n # Collect all nodes: keys and prerequisites\n all_nodes = set(graph.keys())\n for prereqs in graph.values():\n all_nodes.update(prereqs)\n\n # Build remaining prerequisites for each node (as a set)\n remaining_prereqs = {node: set() for node in all_nodes}\n for node, prereqs in graph.items():\n remaining_prereqs[node] = set(prereqs)\n\n # Build reverse mapping: for each node, the set of nodes that depend on it\n prereq_map = {node: set() for node in all_nodes}\n for node, prereqs in graph.items():\n for p in prereqs:\n prereq_map[p].add(node)\n\n # Initialize available nodes (no prerequisites)\n available = set()\n for node in all_nodes:\n if len(remaining_prereqs[node]) == 0:\n available.add(node)\n\n batches = []\n while available:\n batch = sorted(available)\n batches.append(batch)\n\n # Update prerequisites of dependents\n for node in batch:\n for dependent in prereq_map[node]:\n remaining_prereqs[dependent].remove(node)\n if len(remaining_prereqs[dependent]) == 0:\n available.add(dependent)\n\n # Remove processed nodes from available\n available = available - batch\n\n # Check if all nodes were processed (no cycles)\n processed = set()\n for batch in batches:\n processed.update(batch)\n if processed != all_nodes:\n raise ValueError(\"Cycle detected\")\n\n return batches\n```"},{"model":"minicpm","task":"sql","phase":"retry","budget":8192,"passed":false,"score":0,"content_passed":false,"tokens":8192,"phase_s":191.70123149990107,"decode_s":191.59962254192217,"prefill_s":0.10160895797889681,"decode_tps":42.75582535768088,"wall_s":195.83959233295172,"wall_kind":"fresh command","finish":"length","peak_memory_bytes":5504983064,"load_s":0.5241361670196056,"strict_pass":false,"accepted_pass":false,"fence_removed":false,"interrupted":false,"final_text_present":false,"delivered_final":false,"display_tps":0.0,"correct_answer_tps":0.0,"outcome":"incomplete","final_answer":""},{"model":"minicpm","task":"cache","phase":"retry","budget":8192,"passed":true,"score":100.0,"content_passed":true,"tokens":5549,"phase_s":123.07932483300101,"decode_s":122.86864975001663,"prefill_s":0.2106750829843804,"decode_tps":45.16204915810307,"wall_s":126.73894325003494,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5477325032,"load_s":0.7708993330597878,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":45.16204915810307,"correct_answer_tps":45.16204915810307,"outcome":"pass","final_answer":"{\n \"read_results\": [10, null, 30, null, 31, null, null, 70],\n \"final_lru_to_mru\": [\"F\", \"H\", \"G\"],\n \"hits\": 4,\n \"misses\": 4\n}"},{"model":"minicpm","task":"extract","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":3010,"phase_s":65.45775062509347,"decode_s":65.31304254208226,"prefill_s":0.14470808301120996,"decode_tps":46.08574157390705,"wall_s":69.34674516704399,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5352052252,"load_s":0.6402295840671286,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":46.08574157390705,"correct_answer_tps":46.08574157390705,"outcome":"pass","final_answer":"[\n {\"id\": \"S-17\", \"status\": \"departed\", \"units\": 10, \"departed_utc\": \"2026-05-04T22:10:00Z\", \"evidence\": [\"L3\", \"L5\"]},\n {\"id\": \"S-18\", \"status\": \"planned\", \"units\": 9, \"departed_utc\": null, \"evidence\": [\"L7\"]},\n {\"id\": \"S-19\", \"status\": \"departed\", \"units\": 5, \"departed_utc\": \"2026-05-05T02:50:00Z\", \"evidence\": [\"L6\"]},\n {\"id\": \"S-20\", \"status\": \"cancelled\", \"units\": 0, \"departed_utc\": null, \"evidence\": [\"L8\"]}\n]"},{"model":"minicpm","task":"policy","phase":"original","budget":4096,"passed":true,"score":100.0,"content_passed":true,"tokens":1551,"phase_s":33.87919691600837,"decode_s":33.7387983329827,"prefill_s":0.14039858302567154,"decode_tps":45.970813325730056,"wall_s":36.16702595795505,"wall_kind":"fresh command","finish":"stop","peak_memory_bytes":5342041584,"load_s":0.5971827920293435,"strict_pass":true,"accepted_pass":true,"fence_removed":false,"interrupted":false,"final_text_present":true,"delivered_final":true,"display_tps":45.970813325730056,"correct_answer_tps":45.970813325730056,"outcome":"pass","final_answer":"{\n \"A\": {\"eligible\": false, \"credit\": 0},\n \"B\": {\"eligible\": true, \"credit\": 8.5},\n \"C\": {\"eligible\": false, \"credit\": 0},\n \"D\": {\"eligible\": null, \"credit\": null},\n \"E\": {\"eligible\": true, \"credit\": null},\n \"F\": {\"eligible\": false, \"credit\": 0}\n}"}],"cohort":"Historical evaluation; budgets, runtimes and stopping rules vary. Selected latest is not a fresh suite or best-of-n guarantee."}]