{"id": "warmup:0:01-raccoon-refund", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 4.70809999981725e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "warmup:0:02-sarcasm-detector", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.489000000200804e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "warmup:0:03-dungeon-risk", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.470999999815149e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "warmup:0:04-vampire-injection", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 6.455200000132777e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:0:01-raccoon-refund", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.305999999980713e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:0:02-sarcasm-detector", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 5.015099999994277e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:0:03-dungeon-risk", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.47800000003906e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:0:04-vampire-injection", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.8620999998452135e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:1:01-raccoon-refund", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 4.087099999949828e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:1:02-sarcasm-detector", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.856099999666185e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:1:03-dungeon-risk", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.147099999978309e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:1:04-vampire-injection", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.9590000000467853e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:2:01-raccoon-refund", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.183999999871844e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:2:02-sarcasm-detector", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.098999999906482e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:2:03-dungeon-risk", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 1.980100000054108e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:2:04-vampire-injection", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 0.00011722300000016617, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:3:01-raccoon-refund", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.9950999998362704e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:3:02-sarcasm-detector", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.876099999899907e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:3:03-dungeon-risk", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 4.061100000285478e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:3:04-vampire-injection", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.418100000056711e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:4:01-raccoon-refund", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.312100000168016e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:4:02-sarcasm-detector", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 1.9251000001219154e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:4:03-dungeon-risk", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 3.147000000325306e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "example:4:04-vampire-injection", "ok": false, "error": "ValueError: Choice requires between 1 and 255 candidates", "server_wall_seconds": 2.0701000000400427e-05, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-01-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.004546356015016495}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 110, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 1.462729136, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.4628509990000005, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-01-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0018351383893756156}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15340106800000086, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15349537999999896, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-02-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9582067904909487}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15482835099999903, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15493629399999875, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-02-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9234949179076533}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 98, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17701193900000334, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1773623569999998, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-03-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.19559769653889897}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15570983199999944, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15580055399999893, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-03-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.7767047476391329}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 100, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16497443399999767, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.165065405, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-04-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.006097559246187505}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 107, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15202277499999894, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15210986700000007, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-04-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0070844349405043295}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 104, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15169144700000103, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15177946900000094, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-05-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.1947865770182771}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 111, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16467994299999944, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16477309500000104, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-05-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.4050861264085924}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15059717200000122, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1506923140000005, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-06-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8928004223563749}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 113, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16451801000000188, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16460911200000083, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-policy-06-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.6023053413313587}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.174721482999999, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17483756499999714, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-01-0", "response": {"answers": {"decision": {"type": "choice", "choice": "other", "probabilities": {"cancel": 0.07818400236766278, "change_address": 0.04363065798892138, "other": 0.5410917020553818, "refund": 0.28871848072870454, "status": 0.04837515685932945}, "confidence": 0.4263646275692272}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 347, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.745377843, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7454884649999975, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-01-1", "response": {"answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.06839601750669007, "change_address": 0.010057394443794183, "other": 0.16815394367268813, "refund": 0.7443518224701553, "status": 0.009040821906672351}, "confidence": 0.6804397780876942}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 352, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7730814280000011, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7732125910000001, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-02-0", "response": {"answers": {"decision": {"type": "choice", "choice": "other", "probabilities": {"cancel": 0.04218634781854709, "change_address": 0.11301234750869132, "other": 0.6405832798938752, "refund": 0.09004592316792623, "status": 0.11417210161096017}, "confidence": 0.550729099867344}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 352, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7688350239999977, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.768944476999998, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-02-1", "response": {"answers": {"decision": {"type": "choice", "choice": "other", "probabilities": {"cancel": 0.052110234470439984, "change_address": 0.01675406082517303, "other": 0.6519213265941786, "refund": 0.06831170128596796, "status": 0.21090267682424044}, "confidence": 0.5649016582427232}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 347, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7111447909999988, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7112405430000024, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-03-0", "response": {"answers": {"decision": {"type": "choice", "choice": "change_address", "probabilities": {"cancel": 0.009454237311202042, "change_address": 0.799176711236621, "other": 0.15775210432690717, "refund": 0.009330530736722294, "status": 0.024286416388547272}, "confidence": 0.7489708890457764}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 357, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7352959249999991, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7353946570000005, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-03-1", "response": {"answers": {"decision": {"type": "choice", "choice": "change_address", "probabilities": {"cancel": 0.014825455526661353, "change_address": 0.7790067886582849, "other": 0.1666151664542757, "refund": 0.015091116120253642, "status": 0.024461473240524473}, "confidence": 0.723758485822856}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 347, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7259181300000002, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7260165930000007, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-04-0", "response": {"answers": {"decision": {"type": "choice", "choice": "cancel", "probabilities": {"cancel": 0.7226179063467686, "change_address": 0.009954422167293695, "other": 0.25125446165585796, "refund": 0.0072121293400606865, "status": 0.008961080490019127}, "confidence": 0.6532723829334608}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 372, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.6873650439999999, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6874583459999997, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-04-1", "response": {"answers": {"decision": {"type": "choice", "choice": "cancel", "probabilities": {"cancel": 0.8039515165323793, "change_address": 0.005293826343554094, "other": 0.18087791015889443, "refund": 0.004046187329307621, "status": 0.005830559635864643}, "confidence": 0.754939395665474}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 347, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.6751535289999993, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6752491910000025, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-05-0", "response": {"answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.07823923508342942, "change_address": 0.039371806693947534, "other": 0.36130049224252775, "refund": 0.48641953497906737, "status": 0.03466893100102797}, "confidence": 0.35802441872383417}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 347, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7140245960000016, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7141253079999998, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-05-1", "response": {"answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.130457802805025, "change_address": 0.04298073600148193, "other": 0.29938354557379643, "refund": 0.48524822063928263, "status": 0.0419296949804139}, "confidence": 0.3565602757991033}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 342, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7039336749999983, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7040344269999999, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-06-0", "response": {"answers": {"decision": {"type": "choice", "choice": "other", "probabilities": {"cancel": 0.06597122259095, "change_address": 0.03651251192584452, "other": 0.5873489556549019, "refund": 0.08564356058711635, "status": 0.22452374924118734}, "confidence": 0.48418619456862727}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 337, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7559144749999973, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7560896889999995, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-intent-06-1", "response": {"answers": {"decision": {"type": "choice", "choice": "other", "probabilities": {"cancel": 0.044496135484351135, "change_address": 0.03585650522719127, "other": 0.6561896359306304, "refund": 0.10806591371235762, "status": 0.15539180964546967}, "confidence": 0.570237044913288}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 362, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7608955839999965, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.760988066000003, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-01-0", "response": {"answers": {"decision": {"type": "score", "score": 0.005897072792583899, "probabilities": {"0": 0.996625997306072, "1": 0.00110441372662887, "2": 0.002016107835942284, "3": 0.0002534811313568203}, "confidence": 0.9941029272074161, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 277, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5659865840000009, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5660878660000037, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-01-1", "response": {"answers": {"decision": {"type": "score", "score": 0.02646018691385328, "probabilities": {"0": 0.9808739799490174, "1": 0.012409362153476669, "2": 0.006099148932141521, "3": 0.0006175089653645235}, "confidence": 0.9735398130861467, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 281, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5850367570000046, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5851488800000055, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-02-0", "response": {"answers": {"decision": {"type": "score", "score": 0.9787646393790179, "probabilities": {"0": 0.11660422081432925, "1": 0.7952617635838353, "2": 0.08089917101032397, "3": 0.007234844591511591}, "confidence": 0.7880269189923236, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 293, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5683842569999982, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.568490618999995, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-02-1", "response": {"answers": {"decision": {"type": "score", "score": 0.8871465042805728, "probabilities": {"0": 0.1286363352939144, "1": 0.8608845602364638, "2": 0.005175369364755892, "3": 0.005303735104865792}, "confidence": 0.855580825131598, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 297, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5975489130000042, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5976961069999973, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-03-0", "response": {"answers": {"decision": {"type": "score", "score": 1.4862656072371796, "probabilities": {"0": 0.2508590317611235, "1": 0.02125706617633954, "2": 0.7186431651267711, "3": 0.009240736935765981}, "confidence": 0.4677841333656475, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 281, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6242625029999971, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6243753150000018, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-03-1", "response": {"answers": {"decision": {"type": "score", "score": 1.060240343736209, "probabilities": {"0": 0.4522263231333461, "1": 0.05232828821102916, "2": 0.4784241104416946, "3": 0.017021278213930225}, "confidence": 0.026197787308348652, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 289, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5792098320000036, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5793344539999978, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-04-0", "response": {"answers": {"decision": {"type": "score", "score": 2.83729592034634, "probabilities": {"0": 0.049653040173716, "1": 0.004174078760133623, "2": 0.0053968016122442975, "3": 0.9407760794539061}, "confidence": 0.8372959203463405, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 289, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6832370879999985, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6833545510000008, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-04-1", "response": {"answers": {"decision": {"type": "score", "score": 2.954996454202114, "probabilities": {"0": 0.011457250373925877, "1": 0.0017930936965897465, "2": 0.007045607282928684, "3": 0.9797040486465557}, "confidence": 0.9549964542021142, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 285, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5919243310000013, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5920486340000011, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-05-0", "response": {"answers": {"decision": {"type": "score", "score": 0.07067216038864432, "probabilities": {"0": 0.9625253840680446, "1": 0.014335296850055072, "2": 0.01308109370711198, "3": 0.010058225374788428}, "confidence": 0.9293278396113557, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 301, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6111020280000048, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6112017399999985, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-05-1", "response": {"answers": {"decision": {"type": "score", "score": 0.03588462575372401, "probabilities": {"0": 0.9818102986470459, "1": 0.0026975743348925008, "2": 0.01328932963535308, "3": 0.0022027973827084503}, "confidence": 0.964115374246276, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 297, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6030070079999987, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6031111400000029, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-06-0", "response": {"answers": {"decision": {"type": "score", "score": 1.9034370719452818, "probabilities": {"0": 0.03678694445461755, "1": 0.023697712089138057, "2": 0.9388066705125896, "3": 0.000708672943654871}, "confidence": 0.902019726057972, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 301, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5644924579999966, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5645923800000006, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-ordinal-06-1", "response": {"answers": {"decision": {"type": "score", "score": 1.9683484712220773, "probabilities": {"0": 0.010363804440369958, "1": 0.01167196597519994, "2": 0.9772161835064127, "3": 0.0007480460780173314}, "confidence": 0.9668523790660428, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 329, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6439548159999973, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6440596690000007, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-01-0", "response": {"answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.33635063861277037, "pickup": 0.18089323999315143, "post": 0.25824607452670995, "unknown": 0.2245100468673682}, "confidence": 0.11513418481702715}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 294, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.585579580000001, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5856721719999953, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-01-1", "response": {"answers": {"decision": {"type": "choice", "choice": "pickup", "probabilities": {"courier": 0.03379527319890452, "pickup": 0.6522939302728242, "post": 0.1280212944084394, "unknown": 0.18588950211983193}, "confidence": 0.5363919070304323}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 302, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6172488030000025, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6174258379999955, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-02-0", "response": {"answers": {"decision": {"type": "choice", "choice": "unknown", "probabilities": {"courier": 0.02540073497203303, "pickup": 0.025434462606163822, "post": 0.15563065727735986, "unknown": 0.7935341451444433}, "confidence": 0.7247121935259244}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 306, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6127315089999996, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6128246209999944, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-02-1", "response": {"answers": {"decision": {"type": "choice", "choice": "unknown", "probabilities": {"courier": 0.02831718998513974, "pickup": 0.027876719047348647, "post": 0.04522002106905781, "unknown": 0.8985860698984539}, "confidence": 0.8647814265312719}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 302, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6142887220000048, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6144194540000001, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-03-0", "response": {"answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.5309792313601489, "pickup": 0.12750848154326838, "post": 0.21305005157783805, "unknown": 0.1284622355187447}, "confidence": 0.3746389751468652}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 318, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6425554679999976, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6426694710000049, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-03-1", "response": {"answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.8281999217794679, "pickup": 0.0295680953541492, "post": 0.11550170827517826, "unknown": 0.026730274591204674}, "confidence": 0.7709332290392905}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 314, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5924941390000029, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5926552430000029, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-04-0", "response": {"answers": {"decision": {"type": "choice", "choice": "post", "probabilities": {"courier": 0.08941580432656569, "pickup": 0.033228712660862414, "post": 0.6891621901295355, "unknown": 0.18819329288303638}, "confidence": 0.5855495868393806}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 306, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6381002549999977, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.638197556999998, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-04-1", "response": {"answers": {"decision": {"type": "choice", "choice": "post", "probabilities": {"courier": 0.060390229580827036, "pickup": 0.01884620583304756, "post": 0.8552856176436644, "unknown": 0.06547794694246102}, "confidence": 0.8070474901915525}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 294, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6013216170000035, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6014480999999989, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-05-0", "response": {"answers": {"decision": {"type": "choice", "choice": "unknown", "probabilities": {"courier": 0.01423353976629996, "pickup": 0.051857731113646124, "post": 0.030089927955715873, "unknown": 0.9038188011643381}, "confidence": 0.8717584015524508}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 306, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5671829000000059, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5672719119999954, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-05-1", "response": {"answers": {"decision": {"type": "choice", "choice": "unknown", "probabilities": {"courier": 0.21025737105378176, "pickup": 0.07810951057772744, "post": 0.139454142125105, "unknown": 0.5721789762433858}, "confidence": 0.42957196832451433}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 290, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6191469769999998, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6193044499999942, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-06-0", "response": {"answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.9349843456027508, "pickup": 0.017967840238836356, "post": 0.01750348939137009, "unknown": 0.02954432476704275}, "confidence": 0.9133124608036677}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 306, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6389207189999979, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6390196920000051, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-extraction-06-1", "response": {"answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.8428514520562359, "pickup": 0.037971377634529374, "post": 0.04330443320747206, "unknown": 0.07587273710176266}, "confidence": 0.7904686027416479}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 302, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6181251260000025, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6183376199999984, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-01-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9331247871616675}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15737560900000602, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15746592099999646, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-01-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8841112307160623}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15714828400000158, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15729040699999786, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-02-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.3989677218936795}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15273458299999731, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15282081499999833, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-02-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.06170120484158542}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15122674900000277, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15132470100000006, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-03-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.936981938606087}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 92, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.14764240599999567, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.14772701799999766, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-03-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8296029715556033}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15455672400000253, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15471021699999454, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-04-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.6035664952644514}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.158871843, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15896334499999654, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-04-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.5869200353245649}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 93, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15249728699999565, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15258485900000096, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-05-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.661869126771171}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 85, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15293706700000342, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15302184800000163, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-05-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.44371224309711893}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 86, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15091904100000164, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15101591300000194, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-06-0", "response": {"answers": {"decision": {"type": "noul", "noul": 0.7869169783158885}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 101, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16466086500000188, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16475095700000253, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-adequacy-06-1", "response": {"answers": {"decision": {"type": "noul", "noul": 0.4261098198149216}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 103, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15871645, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1588101420000001, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-01-0", "response": {"answers": {"decision": {"type": "choice", "choice": "math", "probabilities": {"coding": 0.023369659170405576, "coding_agent": 0.01570567863295688, "document": 0.019739416207631456, "general": 0.07333267413725539, "math": 0.8238211201867137, "tools": 0.04403145166503703}, "confidence": 0.7885853442240565}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 477, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8511846339999991, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8512893959999985, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-01-1", "response": {"answers": {"decision": {"type": "choice", "choice": "math", "probabilities": {"coding": 0.026813925813817405, "coding_agent": 0.018339333814268572, "document": 0.022045062185240678, "general": 0.07647851132172212, "math": 0.8057342608192074, "tools": 0.05058890604574383}, "confidence": 0.7668811129830488}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 453, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.9447664219999936, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.9449048849999997, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-02-0", "response": {"answers": {"decision": {"type": "choice", "choice": "coding", "probabilities": {"coding": 0.6691233024818956, "coding_agent": 0.09308318532514202, "document": 0.01417947963283543, "general": 0.14334473559641014, "math": 0.01148959970259574, "tools": 0.06877969726112113}, "confidence": 0.6029479629782747}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 483, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8692082580000005, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8693021200000004, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-02-1", "response": {"answers": {"decision": {"type": "choice", "choice": "coding", "probabilities": {"coding": 0.6001045448825174, "coding_agent": 0.08594786528576434, "document": 0.029478180275321154, "general": 0.15964506644311846, "math": 0.020201165127184572, "tools": 0.10462317798609397}, "confidence": 0.5201254538590209}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 447, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8631488259999998, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8632518979999944, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-03-0", "response": {"answers": {"decision": {"type": "choice", "choice": "coding_agent", "probabilities": {"coding": 0.006769810369809449, "coding_agent": 0.856820455640646, "document": 0.018438992063329107, "general": 0.05821191643444084, "math": 0.008272707100926216, "tools": 0.051486118390848434}, "confidence": 0.8281845467687752}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 477, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.845191468000003, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8452884310000002, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-03-1", "response": {"answers": {"decision": {"type": "choice", "choice": "coding_agent", "probabilities": {"coding": 0.00654331649633052, "coding_agent": 0.8851131339879774, "document": 0.01545913556089549, "general": 0.05687770924757802, "math": 0.006019992291246549, "tools": 0.02998671241597195}, "confidence": 0.8621357607855729}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 489, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8594294930000004, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8595296049999988, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-04-0", "response": {"answers": {"decision": {"type": "choice", "choice": "coding_agent", "probabilities": {"coding": 0.019092045142876393, "coding_agent": 0.8596073668048319, "document": 0.016010051907727203, "general": 0.06908503112346356, "math": 0.006949829184274016, "tools": 0.02925567583682695}, "confidence": 0.8315288401657982}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 453, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8939531420000009, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8940489449999944, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-04-1", "response": {"answers": {"decision": {"type": "choice", "choice": "coding_agent", "probabilities": {"coding": 0.029677217580909288, "coding_agent": 0.7830535188395856, "document": 0.02739752656588689, "general": 0.09250414810003971, "math": 0.017872737497970137, "tools": 0.049494851415608415}, "confidence": 0.7396642226075028}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 447, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.898369387999999, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8984680820000008, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-05-0", "response": {"answers": {"decision": {"type": "choice", "choice": "tools", "probabilities": {"coding": 0.028980878404569328, "coding_agent": 0.03029621837154213, "document": 0.08016590527918314, "general": 0.1799051010225692, "math": 0.02218430688279154, "tools": 0.6584675900393447}, "confidence": 0.5901611080472137}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 459, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8986448940000002, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8987451660000048, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-05-1", "response": {"answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.06440866105038784, "coding_agent": 0.1943567688033196, "document": 0.12022474110704941, "general": 0.35209269295892515, "math": 0.0377628724113526, "tools": 0.23115426366896552}, "confidence": 0.22251123155071018}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 459, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8766508040000005, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8768614889999995, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-06-0", "response": {"answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.0683779710336535, "coding_agent": 0.05125631741250466, "document": 0.09530346600220635, "general": 0.4957144595670155, "math": 0.1312673854095102, "tools": 0.15808040057510978}, "confidence": 0.39485735148041856}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 453, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.947028625999998, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.9471316990000034, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:original-routing-06-1", "response": {"answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.08814304068569512, "coding_agent": 0.06935811459934427, "document": 0.10564779258992947, "general": 0.43156502945870945, "math": 0.1281426812058431, "tools": 0.1771433414604786}, "confidence": 0.3178780353504514}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 447, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 0.8548679080000028, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8549717499999971, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-00", "response": {"answers": {"decision": {"type": "choice", "choice": "track_order", "probabilities": {"billing_question": 0.00629483891888552, "cancel_order": 0.02696095967556538, "change_address": 0.0052299254238450206, "report_damage": 0.0025099264735913194, "track_order": 0.9590043495081126}, "confidence": 0.9487554368851409}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 359, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7727686109999965, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7728748830000001, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-01", "response": {"answers": {"decision": {"type": "choice", "choice": "cancel_order", "probabilities": {"billing_question": 0.002104058831530386, "cancel_order": 0.994948529022335, "change_address": 0.001395688663556433, "report_damage": 0.0006395710558495245, "track_order": 0.000912152426728557}, "confidence": 0.9936856612779188}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 359, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7359862969999966, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7360915089999978, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-02", "response": {"answers": {"decision": {"type": "choice", "choice": "change_address", "probabilities": {"billing_question": 0.015535141504311033, "cancel_order": 0.014375470228608628, "change_address": 0.9536935293195136, "report_damage": 0.0034877659797014626, "track_order": 0.012908092967865381}, "confidence": 0.9421169116493918}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 374, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7543133220000016, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7544425550000042, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-03", "response": {"answers": {"decision": {"type": "choice", "choice": "report_damage", "probabilities": {"billing_question": 0.004873892844795531, "cancel_order": 0.05789522779624476, "change_address": 0.0043924110805534, "report_damage": 0.9113821060466963, "track_order": 0.021456362231709953}, "confidence": 0.8892276325583703}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 319, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7678733199999996, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7679700220000001, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-04", "response": {"answers": {"decision": {"type": "choice", "choice": "billing_question", "probabilities": {"billing_question": 0.966226980706777, "cancel_order": 0.007877587969882164, "change_address": 0.0065236204894226005, "report_damage": 0.00672506091117131, "track_order": 0.012646749922746838}, "confidence": 0.9577837258834715}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 344, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7382090689999998, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.738360772, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-05", "response": {"answers": {"decision": {"type": "choice", "choice": "track_order", "probabilities": {"billing_question": 0.006906245088912445, "cancel_order": 0.017129456909077603, "change_address": 0.009583420142721321, "report_damage": 0.005231558450094299, "track_order": 0.9611493194091942}, "confidence": 0.9514366492614929}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 339, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7423985280000025, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7425241509999978, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-06", "response": {"answers": {"decision": {"type": "choice", "choice": "set_alarm", "probabilities": {"play_music": 0.032539317137102286, "send_message": 0.05060791764422828, "set_alarm": 0.8612658779104091, "turn_off_lights": 0.030847604954620328, "weather": 0.024739282353639994}, "confidence": 0.8265823473880113}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 327, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7408161180000121, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7409093109999958, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-07", "response": {"answers": {"decision": {"type": "choice", "choice": "play_music", "probabilities": {"play_music": 0.5343717384414057, "send_message": 0.18582171101926678, "set_alarm": 0.08747786724988597, "turn_off_lights": 0.0993164645223643, "weather": 0.09301221876707712}, "confidence": 0.4179646730517573}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 292, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7154170700000009, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.71552806199999, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-08", "response": {"answers": {"decision": {"type": "choice", "choice": "weather", "probabilities": {"play_music": 0.013383496529568138, "send_message": 0.034279158842319093, "set_alarm": 0.01824946064497631, "turn_off_lights": 0.023638969464734767, "weather": 0.9104489145184017}, "confidence": 0.888061143148002}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 302, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7127390480000031, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7128302499999961, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-09", "response": {"answers": {"decision": {"type": "choice", "choice": "send_message", "probabilities": {"play_music": 0.09008104726763082, "send_message": 0.584131509943622, "set_alarm": 0.16022901378551516, "turn_off_lights": 0.07635103499451873, "weather": 0.08920739400871346}, "confidence": 0.4801643874295274}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 317, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7175082070000087, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7176075590000011, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-10", "response": {"answers": {"decision": {"type": "choice", "choice": "turn_off_lights", "probabilities": {"play_music": 0.035085000670756275, "send_message": 0.015079437971577526, "set_alarm": 0.02417107849128947, "turn_off_lights": 0.9139262125861913, "weather": 0.011738270280185346}, "confidence": 0.8924077657327392}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 307, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7294634600000052, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7295625419999965, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-intent-11", "response": {"answers": {"decision": {"type": "choice", "choice": "set_alarm", "probabilities": {"play_music": 0.02045669672272571, "send_message": 0.015237545645735371, "set_alarm": 0.938837593550628, "turn_off_lights": 0.015767564482344273, "weather": 0.009700599598566645}, "confidence": 0.9235469919382849}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 307, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7131834609999999, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.713283993999994, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-00", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9810170676683818}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17054793999999163, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17094387999999583, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-01", "response": {"answers": {"decision": {"type": "noul", "noul": 0.001216523316912468}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17262399500000924, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17271922700000175, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-02", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9847554295182771}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 96, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16750487800000258, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16763501100000155, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-03", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0011313368338833703}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 99, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15315354999999897, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15325850300000354, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-04", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9809204328238449}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 88, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15403727999999717, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15413559200000293, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-05", "response": {"answers": {"decision": {"type": "noul", "noul": 0.001976710848071371}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15207480600000167, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.152168477999993, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-06", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9798633174653026}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 109, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15513474499999802, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15526101799999026, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-07", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0011227073338304894}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 84, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15218131100000676, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1522683820000026, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9838828225494912}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16451260399999512, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16461061599999027, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-09", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0017795491591166767}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 80, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.154138856000003, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15423611899998946, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-10", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9777758167176331}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 89, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15469564899999, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15478623100000277, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-fact-11", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0010537590313325147}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 90, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17242073500000288, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17253073700000243, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-00", "response": {"answers": {"decision": {"type": "choice", "choice": "paypal", "probabilities": {"bank_transfer": 0.016245509218015236, "cash": 0.0070134416252111914, "credit_card": 0.01833500720141554, "paypal": 0.9584060419553581}, "confidence": 0.9445413892738107}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 224, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5820208869999988, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5823859050000095, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-01", "response": {"answers": {"decision": {"type": "choice", "choice": "cash", "probabilities": {"bank_transfer": 0.003301158262566514, "cash": 0.9914802327179952, "credit_card": 0.0022246415995397866, "paypal": 0.0029939674198984356}, "confidence": 0.9886403102906604}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 232, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5931507579999931, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5933244819999999, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-02", "response": {"answers": {"decision": {"type": "choice", "choice": "bank_transfer", "probabilities": {"bank_transfer": 0.9461716218713028, "cash": 0.017755290654454508, "credit_card": 0.016477155258232384, "paypal": 0.01959593221601037}, "confidence": 0.928228829161737}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 236, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5401677350000114, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5405413449999941, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-03", "response": {"answers": {"decision": {"type": "choice", "choice": "high", "probabilities": {"high": 0.9544196121611135, "low": 0.00045451769138430347, "medium": 0.0009036798781106869, "urgent": 0.04422219026939157}, "confidence": 0.9392261495481513}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 258, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.574941999999993, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5750481130000082, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-04", "response": {"answers": {"decision": {"type": "choice", "choice": "low", "probabilities": {"high": 0.0006886069692984683, "low": 0.9976860604550456, "medium": 0.0010190741324985811, "urgent": 0.0006062584431573328}, "confidence": 0.9969147472733941}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5275872570000075, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5279974660000022, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-05", "response": {"answers": {"decision": {"type": "choice", "choice": "urgent", "probabilities": {"high": 0.07212902718507021, "low": 0.0011845498303778115, "medium": 0.001611978442306138, "urgent": 0.9250744445422459}, "confidence": 0.9000992593896612}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5579339869999984, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5580271390000036, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-06", "response": {"answers": {"decision": {"type": "choice", "choice": "M", "probabilities": {"L": 0.02424764814430783, "M": 0.939064745174826, "S": 0.024908805409049224, "XL": 0.011778801271816956}, "confidence": 0.9187529935664346}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 237, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5980625880000048, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5981654400000025, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-07", "response": {"answers": {"decision": {"type": "choice", "choice": "XL", "probabilities": {"L": 0.04385937776820933, "M": 0.017313363283543697, "S": 0.015870224300212066, "XL": 0.9229570346480348}, "confidence": 0.8972760461973799}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 229, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5676251689999958, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5677126709999953, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-08", "response": {"answers": {"decision": {"type": "choice", "choice": "thursday", "probabilities": {"friday": 0.02963589260877429, "monday": 0.04031876227506881, "thursday": 0.843477960674642, "tuesday": 0.03996647101946457, "wednesday": 0.046600913422050395}, "confidence": 0.8043474508433024}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 293, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.720431015999992, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7205218880000075, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-09", "response": {"answers": {"decision": {"type": "choice", "choice": "monday", "probabilities": {"friday": 0.09462073978775477, "monday": 0.6052091330643592, "thursday": 0.08670472189581048, "tuesday": 0.10919395009630554, "wednesday": 0.10427145515577012}, "confidence": 0.5065114163304489}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 303, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7047795760000071, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7048773869999962, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-10", "response": {"answers": {"decision": {"type": "choice", "choice": "friday", "probabilities": {"friday": 0.8311136782629532, "monday": 0.03936769989473741, "thursday": 0.041680679984600184, "tuesday": 0.038732337474571076, "wednesday": 0.049105604383138095}, "confidence": 0.7888920978286914}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 283, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7045778560000002, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7046750180000032, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-extraction-11", "response": {"answers": {"decision": {"type": "choice", "choice": "credit_card", "probabilities": {"bank_transfer": 0.04985203970311205, "cash": 0.0457267796994118, "credit_card": 0.8351338018588117, "paypal": 0.0692873787386644}, "confidence": 0.7801784024784156}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 248, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5915986220000065, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5916906740000059, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-00", "response": {"answers": {"decision": {"type": "choice", "choice": "get_weather", "probabilities": {"convert_currency": 0.02011641530899163, "create_calendar_event": 0.02405353776168441, "get_weather": 0.8788432163649198, "send_email": 0.0339801231111114, "translate_text": 0.04300670745329277}, "confidence": 0.8485540204561498}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 330, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7147300459999997, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7148347690000065, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-01", "response": {"answers": {"decision": {"type": "choice", "choice": "send_email", "probabilities": {"convert_currency": 0.08444431592182232, "create_calendar_event": 0.07721413124450834, "get_weather": 0.07102140545545148, "send_email": 0.6750713251315832, "translate_text": 0.09224882224663475}, "confidence": 0.5938391564144789}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 335, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7389451469999955, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7390370090000005, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-02", "response": {"answers": {"decision": {"type": "choice", "choice": "create_calendar_event", "probabilities": {"convert_currency": 0.004745540548936337, "create_calendar_event": 0.9449315599881751, "get_weather": 0.008107397322109761, "send_email": 0.0347930321200573, "translate_text": 0.0074224700207212784}, "confidence": 0.9311644499852192}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 350, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7132746270000041, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7133914989999965, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-03", "response": {"answers": {"decision": {"type": "choice", "choice": "convert_currency", "probabilities": {"convert_currency": 0.8663964997738783, "create_calendar_event": 0.01010423097517344, "get_weather": 0.013769808987039172, "send_email": 0.016847431073668564, "translate_text": 0.09288202919024047}, "confidence": 0.8329956247173478}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 340, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7161424979999964, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7162362609999917, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-04", "response": {"answers": {"decision": {"type": "choice", "choice": "translate_text", "probabilities": {"convert_currency": 0.023631526829562804, "create_calendar_event": 0.03567449034892053, "get_weather": 0.05200930186759246, "send_email": 0.07667207088192987, "translate_text": 0.8120126100719943}, "confidence": 0.7650157625899928}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 320, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.6928574809999901, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6929574140000057, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-05", "response": {"answers": {"decision": {"type": "choice", "choice": "create_calendar_event", "probabilities": {"convert_currency": 0.028842207662865056, "create_calendar_event": 0.7673719960659277, "get_weather": 0.034271297787604835, "send_email": 0.13461211619070196, "translate_text": 0.0349023822929006}, "confidence": 0.7092149950824091}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 350, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.714845964999995, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7149676370000009, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-06", "response": {"answers": {"decision": {"type": "choice", "choice": "search_flights", "probabilities": {"book_restaurant": 0.021304903460562036, "call_taxi": 0.02712023095183556, "get_stock_price": 0.01706614932971986, "search_flights": 0.9154719541639353, "set_reminder": 0.019036762093947116}, "confidence": 0.8943399427049192}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 350, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7278392279999935, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7279616210000057, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-07", "response": {"answers": {"decision": {"type": "choice", "choice": "book_restaurant", "probabilities": {"book_restaurant": 0.8482806412256484, "call_taxi": 0.06945456100311884, "get_stock_price": 0.018449416361981583, "search_flights": 0.022322316226068197, "set_reminder": 0.04149306518318307}, "confidence": 0.8103508015320604}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 350, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.726922572999996, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7270167650000019, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-08", "response": {"answers": {"decision": {"type": "choice", "choice": "set_reminder", "probabilities": {"book_restaurant": 0.020371297322743648, "call_taxi": 0.025688773324110032, "get_stock_price": 0.008206679612230854, "search_flights": 0.011013515879438456, "set_reminder": 0.934719733861477}, "confidence": 0.9183996673268463}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 345, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7356253609999897, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7357310040000016, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-09", "response": {"answers": {"decision": {"type": "choice", "choice": "get_stock_price", "probabilities": {"book_restaurant": 0.03405185767833813, "call_taxi": 0.0419264180722505, "get_stock_price": 0.8739746354959743, "search_flights": 0.029895280806924572, "set_reminder": 0.02015180794651246}, "confidence": 0.842468294369968}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 330, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.6999870949999973, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7000864179999979, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-10", "response": {"answers": {"decision": {"type": "choice", "choice": "call_taxi", "probabilities": {"book_restaurant": 0.015436890989094565, "call_taxi": 0.948583200003435, "get_stock_price": 0.009180774646596538, "search_flights": 0.015977530329532974, "set_reminder": 0.010821604031340838}, "confidence": 0.9357290000042939}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 330, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7704387359999885, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7705368590000035, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:easy-tool_selection-11", "response": {"answers": {"decision": {"type": "choice", "choice": "set_reminder", "probabilities": {"book_restaurant": 0.00964984045274769, "call_taxi": 0.017407060308645736, "get_stock_price": 0.0067975925896667285, "search_flights": 0.007571625840488616, "set_reminder": 0.9585738808084512}, "confidence": 0.948217351010564}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 345, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7157622100000083, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7158578819999946, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-01", "response": {"answers": {"decision": {"type": "choice", "choice": "deny_vacancy_exclusion", "probabilities": {"deny_repeated_seepage": 0.1431882075861284, "deny_vacancy_exclusion": 0.2771998047019216, "pay_full_estimate_less_deductible": 0.13361603976101166, "pay_subject_to_10000_sublimit": 0.18353916018095698, "pay_subject_to_15000_sublimit": 0.2624567877699813}, "confidence": 0.09649975587740202}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 18897, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 2.2491180859999957, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 2.249380142000007, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-04", "response": {"answers": {"decision": {"type": "choice", "choice": "department_head", "probabilities": {"budget_holder": 0.026613501968795, "cfo": 0.026046222134025694, "department_head": 0.9039814794943896, "executive_committee": 0.024830843087329628, "vp_and_finance_director": 0.018527953315460064}, "confidence": 0.8799768493679868}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 12437, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.7622905199999934, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.7624696559999933, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-08", "response": {"answers": {"decision": {"type": "choice", "choice": "not_covered", "probabilities": {"full_repair_no_charge": 0.35509309357989016, "not_covered": 0.44479097886941127, "parts_covered_labour_charged": 0.11510229279262423, "pro_rata_replacement_credit": 0.08501363475807433}, "confidence": 0.25972130515921504}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9268, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3005728189999957, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3007502729999914, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-09", "response": {"answers": {"decision": {"type": "choice", "choice": "no_license_required", "probabilities": {"license_exception_lvs": 0.12490147337436053, "license_required": 0.04680435522245256, "no_license_required": 0.8141492664554728, "shipment_prohibited": 0.014144904947714101}, "confidence": 0.7521990219406304}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9820, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.397480485999992, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3976647100000008, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-11", "response": {"answers": {"decision": {"type": "noul", "noul": 0.022210931268566308}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2334, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.3658156649999995, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.3660296400000078, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-13", "response": {"answers": {"decision": {"type": "noul", "noul": 0.005015450741862976}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2426, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.3551381310000039, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.3554537980000134, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-17", "response": {"answers": {"decision": {"type": "choice", "choice": "no_page_ticket_only", "probabilities": {"database_oncall": 0.16967515433827915, "no_page_ticket_only": 0.40416820942215825, "platform_oncall": 0.2708525025777756, "security_incident_response": 0.049280413995812566, "vendor_support": 0.1060237196659743}, "confidence": 0.2552102617776978}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 11412, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.7744758249999961, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.7751551199999938, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-long_policy-19", "response": {"answers": {"decision": {"type": "noul", "noul": 0.2474233765710823}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2325, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.31518587399999376, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.3153508190000025, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-probability-03", "response": {"answers": {"decision": {"type": "noul", "noul": 0.07075016524370736}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 429, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.19253220000000226, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.19263995400000056, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-probability-04", "response": {"answers": {"decision": {"type": "choice", "choice": "late", "probabilities": {"early": 0.11412450673841884, "late": 0.5980471312920042, "on_time": 0.28782836196957695}, "confidence": 0.39707069693800623}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3229, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.7300501999999938, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7301691720000036, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-probability-07", "response": {"answers": {"decision": {"type": "choice", "choice": "escalated", "probabilities": {"escalated": 0.5163887484545369, "reopened": 0.05815022497822051, "resolved_first_contact": 0.4254610265672426}, "confidence": 0.2745831226818053}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3571, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.7463991889999875, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7467731379999947, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-probability-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0862548650697624}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 391, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17344790699999635, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17355105000000037, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-temporal_numeric-03", "response": {"answers": {"decision": {"type": "noul", "noul": 0.024743801884826253}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 487, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.2069069829999961, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.2070194960000009, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-temporal_numeric-06", "response": {"answers": {"decision": {"type": "noul", "noul": 0.6310160281592514}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 522, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.20094454699999176, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.20105709899999624, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-temporal_numeric-07", "response": {"answers": {"decision": {"type": "choice", "choice": "sep_28", "probabilities": {"sep_24": 0.29747571282350405, "sep_25": 0.16995522036670432, "sep_26": 0.07910671074636351, "sep_27": 0.14984299561757847, "sep_28": 0.30361936044584975}, "confidence": 0.12952420055731217}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 6170, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.164193842000003, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.164333604999996, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-temporal_numeric-09", "response": {"answers": {"decision": {"type": "noul", "noul": 0.7189661128999236}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 519, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.1896858470000069, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.18979034900000613, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-a-temporal_numeric-12", "response": {"answers": {"decision": {"type": "score", "score": 2.7820213868142853, "probabilities": {"0": 0.02646940423231934, "1": 0.028426617394193495, "2": 0.08171716570036934, "3": 0.8633868126731178}, "confidence": 0.7820213868142856, "legend": {"0": "No violations", "1": "Exactly one violation", "2": "Exactly two violations", "3": "Three or more violations"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2256, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.7921086850000023, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7922557080000132, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-02", "response": {"answers": {"decision": {"type": "choice", "choice": "real_incident", "probabilities": {"cannot_determine": 0.4541883663472782, "false_positive": 0.07912155440249291, "real_incident": 0.4666900792502289}, "confidence": 0.2000351188753433}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1491, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5443426259999882, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5444609489999976, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-03", "response": {"answers": {"decision": {"type": "choice", "choice": "insufficient_information", "probabilities": {"insufficient_information": 0.8272361245300596, "time_barred": 0.13917611083551665, "within_limitation": 0.03358776463442377}, "confidence": 0.7408541867950893}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1313, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5445808839999984, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5448873509999999, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-07", "response": {"answers": {"decision": {"type": "choice", "choice": "insufficient_information", "probabilities": {"insufficient_information": 0.7799720307631853, "over_limit": 0.1974936692007761, "within_policy": 0.022534300036038485}, "confidence": 0.6699580461447779}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1378, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5503796499999964, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.550526743000006, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-09", "response": {"answers": {"decision": {"type": "choice", "choice": "compromised", "probabilities": {"compromised": 0.42218571088407253, "inconclusive": 0.1926382854849741, "legitimate": 0.38517600363095345}, "confidence": 0.1332785663261088}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1788, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.6229563010000021, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6230798439999887, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-10", "response": {"answers": {"decision": {"type": "choice", "choice": "send_to_alex_moreno", "probabilities": {"ask_user_which_alex": 0.10096295583472818, "send_to_alex_moreno": 0.82879778185083, "send_to_alexandra_lind": 0.07023926231444186}, "confidence": 0.7431966727762449}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1231, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5409549300000123, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5410963530000004, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-11", "response": {"answers": {"decision": {"type": "choice", "choice": "grade_correct", "probabilities": {"cannot_determine": 0.4851944441049207, "grade_correct": 0.49107177661046386, "grade_should_be_higher": 0.02373377928461544}, "confidence": 0.23660766491569588}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1259, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5263561760000073, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5264799480000022, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-ambiguous-13", "response": {"answers": {"decision": {"type": "choice", "choice": "cannot_determine", "probabilities": {"cannot_determine": 0.6871876090863492, "charge_data_team": 0.09761689070163583, "charge_platform_team": 0.1127748658856381, "split_50_50": 0.10242063432637681}, "confidence": 0.5829168121151324}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2129, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.8326384010000112, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8327837739999921, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-multi_hop-03", "response": {"answers": {"decision": {"type": "choice", "choice": "tier2_department_head", "probabilities": {"tier1_team_lead": 0.1226028573635987, "tier2_department_head": 0.502554055978575, "tier3_cfo": 0.22762521054842094, "tier4_board": 0.14721787610940537}, "confidence": 0.3367387413047667}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 11644, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.602481809000011, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.602680544000009, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-multi_hop-04", "response": {"answers": {"decision": {"type": "choice", "choice": "dmitri", "probabilities": {"ana": 0.1445035867082096, "bjorn": 0.21500474830740338, "chen": 0.1440572443462374, "dmitri": 0.28089186451463183, "esra": 0.11300357197196997, "no_page_ticket_only": 0.10253898415154782}, "confidence": 0.1370702374175582}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 13736, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 1.946591615999992, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.9467693199999871, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-multi_hop-05", "response": {"answers": {"decision": {"type": "score", "score": 1.9204153304723675, "probabilities": {"0": 0.06832154256947005, "1": 0.4226135877844848, "2": 0.17710004964770695, "3": 0.184257636600884, "4": 0.14770718339745414}, "confidence": 0.11911798699057707, "legend": {"0": "Level 0 — no award", "1": "Level 1 — EUR 500", "2": "Level 2 — EUR 1,200", "3": "Level 3 — EUR 2,000", "4": "Level 4 — EUR 3,000"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 12623, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.7325092390000094, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.7327027540000017, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-multi_hop-07", "response": {"answers": {"decision": {"type": "choice", "choice": "credit_50_percent", "probabilities": {"credit_10_percent": 0.19135772717354385, "credit_20_percent": 0.1321120834733533, "credit_25_percent": 0.19618255791103467, "credit_50_percent": 0.20529941054904086, "credit_full_month": 0.13510074550677767, "no_credit": 0.1399474753862497}, "confidence": 0.046359292658849034}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 15346, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 2.0407358930000044, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 2.040913066999991, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-multi_hop-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.6042393150899311}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2312, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.3457930309999995, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.3459648460000011, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-probability-01", "response": {"answers": {"decision": {"type": "choice", "choice": "merchant_wins", "probabilities": {"customer_wins": 0.4027429892617908, "customer_withdraws": 0.029027641622248045, "merchant_wins": 0.5682293691159612}, "confidence": 0.35234405367394184}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 5640, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.8602566339999953, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8604253980000038, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-probability-02", "response": {"answers": {"decision": {"type": "choice", "choice": "bad_push", "probabilities": {"bad_push": 0.9790530921968643, "database_hardware": 0.009323578535291237, "upstream_provider": 0.011623329267844453}, "confidence": 0.9685796382952965}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1409, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5193835360000065, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5195002690000052, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-probability-03", "response": {"answers": {"decision": {"type": "noul", "noul": 0.14694630982749252}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1213, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.24579074100000753, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.24603332799999578, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-probability-04", "response": {"answers": {"decision": {"type": "choice", "choice": "attended", "probabilities": {"attended": 0.4729414539223596, "late_cancellation": 0.08833964261705744, "no_show": 0.438718903460583}, "confidence": 0.20941218088353938}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1698, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5354303469999877, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5358316960000025, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-probability-06", "response": {"answers": {"decision": {"type": "choice", "choice": "overturned", "probabilities": {"modified": 0.060911820106937554, "overturned": 0.7282376914844492, "upheld": 0.21085048840861326}, "confidence": 0.5923565372266737}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2974, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.6755588259999996, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6756952189999907, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-tradeoff-01", "response": {"answers": {"decision": {"type": "choice", "choice": "escalate_to_security", "probabilities": {"escalate_to_engineering": 0.0911752186054048, "escalate_to_security": 0.7733021775315237, "page_oncall": 0.012688060906598228, "process_refund": 0.05822998017546304, "send_kb_article": 0.06460456278101023}, "confidence": 0.7166277219144046}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3763, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.0395302889999982, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.0396757219999984, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-tradeoff-03", "response": {"answers": {"decision": {"type": "choice", "choice": "order_b", "probabilities": {"order_a": 0.0941188197797746, "order_b": 0.6222207892067628, "order_c": 0.04495359331261428, "order_d": 0.2387067977008483}, "confidence": 0.4962943856090171}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2809, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.7757518199999964, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7758748029999936, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-tradeoff-06", "response": {"answers": {"decision": {"type": "noul", "noul": 0.06351497804558515}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 790, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.21755555500000412, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.217701488000003, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-tradeoff-07", "response": {"answers": {"decision": {"type": "choice", "choice": "p2_fix_30d", "probabilities": {"out_of_scope": 0.0038263574267177578, "p0_fix_24h": 0.01198530210273051, "p1_fix_7d": 0.01971304791577317, "p2_fix_30d": 0.958773954615143, "p3_backlog": 0.0057013379396356175}, "confidence": 0.9484674432689288}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3129, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.0103349930000007, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.0104834370000049, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-tradeoff-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.07616221610875511}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 601, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.19802491900000518, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.19813847099999293, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-b-tradeoff-12", "response": {"answers": {"decision": {"type": "choice", "choice": "refund_now", "probabilities": {"deny_refund": 0.21388316268280666, "escalate_to_fraud_team": 0.006488502265929509, "offer_store_credit": 0.08560491192877873, "refund_now": 0.6166576357681206, "wait_for_inspection": 0.07736578735436464}, "confidence": 0.5208220447101506}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2620, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.9469395840000061, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.9470692069999984, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-02", "response": {"answers": {"decision": {"type": "choice", "choice": "restrict_all_pending_legal_hold", "probabilities": {"erase_all": 0.021256623651560294, "erase_but_retain_transaction_records": 0.02729070586831904, "erase_marketing_only": 0.007544352981842603, "reject_unverified_request": 0.018613744429968614, "restrict_all_pending_legal_hold": 0.9252945730683094}, "confidence": 0.9066182163353866}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 17370, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 2.106890741000001, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 2.107213877999996, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-03", "response": {"answers": {"decision": {"type": "choice", "choice": "eur_600", "probabilities": {"eur_0": 0.07850814135040189, "eur_150": 0.039653081433608094, "eur_200": 0.08397523981220507, "eur_300": 0.2556888853160174, "eur_400": 0.265513300388629, "eur_600": 0.27666135169913847}, "confidence": 0.13199362203896617}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 20507, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 2.413045671000006, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 2.4132732559999965, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-04", "response": {"answers": {"decision": {"type": "noul", "noul": 0.7520893849801792}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3312, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.423684245000004, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.4239465109999969, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-05", "response": {"answers": {"decision": {"type": "score", "score": 1.3442627914853984, "probabilities": {"0": 0.26585892833883634, "1": 0.20300198862587296, "2": 0.452156446246347, "3": 0.07898263678894384}, "confidence": 0.1862975179075107, "legend": {"0": "Compliant: Named Users do not exceed Licences held on the Measurement Date", "1": "Shortfall of 1 to 5 Licences", "2": "Shortfall of 6 to 15 Licences", "3": "Shortfall of more than 15 Licences"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 12762, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.5278399529999973, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.5280515379999997, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-08", "response": {"answers": {"decision": {"type": "score", "score": 1.9430573653727063, "probabilities": {"0": 0.012042800956445023, "1": 0.06180172253009722, "2": 0.8972107866977644, "3": 0.02894468981569339}, "confidence": 0.8851679857413193, "legend": {"0": "SEV-4 (lowest)", "1": "SEV-3", "2": "SEV-2", "3": "SEV-1 (highest)"}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 12850, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.6031170959999912, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.6033547330000033, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-10", "response": {"answers": {"decision": {"type": "choice", "choice": "credit_minus_restocking_fee", "probabilities": {"credit_minus_restocking_fee": 0.5082159937781418, "full_credit": 0.3329528326190364, "reject_return": 0.10079005203578896, "repair_or_replace_only": 0.05804112156703298}, "confidence": 0.344287991704189}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 14258, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.7270545810000044, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.7272752869999835, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-long_policy-11", "response": {"answers": {"decision": {"type": "noul", "noul": 0.37361041941630907}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3399, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.4069592449999959, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.4073140729999807, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-probability-03", "response": {"answers": {"decision": {"type": "choice", "choice": "halvorsen_polymer", "probabilities": {"halvorsen_polymer": 0.5894412040423541, "kuznets_technik": 0.21257140300525162, "ortega_seal_works": 0.19798739295239434}, "confidence": 0.3841618060635312}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1519, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.5438536160000069, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5439792999999895, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-temporal_numeric-02", "response": {"answers": {"decision": {"type": "noul", "noul": 0.03880125221499484}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 936, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.22004718200000184, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.2201914550000197, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-temporal_numeric-03", "response": {"answers": {"decision": {"type": "choice", "choice": "eur_812_00", "probabilities": {"eur_617_00": 0.19852899426040477, "eur_688_00": 0.22634627218108455, "eur_692_00": 0.1577084322553042, "eur_693_90": 0.17526142825345054, "eur_812_00": 0.24215487304975597}, "confidence": 0.05269359131219495}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3860, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.0203581239999835, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.0204980469999896, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-temporal_numeric-04", "response": {"answers": {"decision": {"type": "choice", "choice": "usd_2340_00", "probabilities": {"usd_2222_50": 0.047823325683723286, "usd_2303_01": 0.24448581790060128, "usd_2303_33": 0.10396230946393231, "usd_2335_00": 0.2829493748887176, "usd_2340_00": 0.3207791720630256}, "confidence": 0.15097396507878197}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 3275, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.016842878999995, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.016967932, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-temporal_numeric-06", "response": {"answers": {"decision": {"type": "choice", "choice": "sep_17_parking", "probabilities": {"none_declined": 0.13761467709528027, "sep_15_office_chair": 0.17375193623215557, "sep_16_taxi": 0.1247295153960268, "sep_17_parking": 0.3478131470127798, "sep_18_book_set": 0.12029959895595545, "sep_21_client_dinner": 0.09579112530780205}, "confidence": 0.2173757764153358}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 5066, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 6, "inference_seconds": 1.3004635119999932, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3006059150000056, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-temporal_numeric-08", "response": {"answers": {"decision": {"type": "choice", "choice": "eur_497_40", "probabilities": {"eur_497_40": 0.2737988826333233, "eur_498_81": 0.15461214623494982, "eur_506_06": 0.15898773274108655, "eur_513_69": 0.20355442169637145, "eur_532_01": 0.209046816694269}, "confidence": 0.09224860329165412}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 4395, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 1.1183170699999891, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.1184682039999814, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-opus-c-temporal_numeric-12", "response": {"answers": {"decision": {"type": "choice", "choice": "covered", "probabilities": {"covered": 0.40147020526008015, "expired_24_month_term": 0.2211115676148284, "expired_30_month_cap": 0.37741822712509154}, "confidence": 0.10220530789012024}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2299, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.6315520659999834, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6316750099999808, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-adversarial-01", "response": {"answers": {"decision": {"type": "choice", "choice": "deny_missing_approval", "probabilities": {"deny_inactive_user": 0.029798589539537727, "deny_missing_approval": 0.5496493731245543, "grant_access": 0.28774606550621007, "grant_owner_exception": 0.13280597182969803}, "confidence": 0.3995324974994056}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 956, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.622194976000003, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6223027689999867, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-adversarial-06", "response": {"answers": {"decision": {"type": "noul", "noul": 0.0029117786335083615}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 241, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.19405658700000572, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1941904010000144, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-adversarial-07", "response": {"answers": {"decision": {"type": "choice", "choice": "retain_until_2031", "probabilities": {"delete_now": 0.01041377667183377, "needs_signature_review": 0.0010452295236809605, "retain_indefinitely": 0.19415293888647167, "retain_until_2031": 0.7943880549180136}, "confidence": 0.7258507398906847}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 975, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6546946899999853, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6549330959999793, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-adversarial-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.2976429079045205}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 218, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.18943231100001867, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1895360639999808, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-adversarial-09", "response": {"answers": {"decision": {"type": "score", "score": 2.5107414728539568, "probabilities": {"0": 0.04654521997338975, "1": 0.03917112836924008, "2": 0.27128061048739394, "3": 0.6430030411699763}, "confidence": 0.5107414728539567, "legend": {"0": "Informational only.", "1": "Routine degradation.", "2": "Major multi-customer degradation.", "3": "Complete outage."}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 804, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6113810709999825, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6115029039999911, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-adversarial-11", "response": {"answers": {"decision": {"type": "choice", "choice": "major_grants", "probabilities": {"ethics_review": 0.15112332191513442, "major_grants": 0.4672481606401473, "manual_hold": 0.3073807924347086, "standard_research": 0.07424772501000969}, "confidence": 0.28966421418686306}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 998, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6898285550000196, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6899674969999978, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-01", "response": {"answers": {"decision": {"type": "choice", "choice": "accept_s2_dusk", "probabilities": {"accept_s2_dusk": 0.5224530034140799, "reject_cutoff": 0.3064523533353695, "reject_tier": 0.08161370538997621, "reroute_s3": 0.08948093786057443}, "confidence": 0.36327067121877316}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9319, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3733334929999899, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3735467779999908, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-02", "response": {"answers": {"decision": {"type": "choice", "choice": "security_primary", "probabilities": {"executive_bridge": 0.004803561177245638, "heron_q2": 0.03217309369615625, "insufficient_information": 0.18412483865027995, "security_primary": 0.7788985064763182}, "confidence": 0.705198008635091}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9449, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.4089183349999814, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.4091423209999903, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-05", "response": {"answers": {"decision": {"type": "choice", "choice": "pay_manager_approved", "probabilities": {"pay_manager_approved": 0.5770865435180798, "reject_category": 0.3617120337675901, "require_director": 0.04002730277680017, "require_finance_board": 0.021174119937529975}, "confidence": 0.43611539135743965}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9125, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3007676960000083, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3010694040000033, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-07", "response": {"answers": {"decision": {"type": "choice", "choice": "deny_prerequisite", "probabilities": {"admit": 0.0371096248663147, "deny_prerequisite": 0.7052038494466242, "deny_score": 0.15138392868977157, "manual_equivalency": 0.1063025969972896}, "confidence": 0.6069384659288323}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9086, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3297055490000105, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.329922143999994, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-08", "response": {"answers": {"decision": {"type": "choice", "choice": "central_engineering", "probabilities": {"central_engineering": 0.713140049018965, "insufficient_log": 0.14617720592629238, "local_maintenance": 0.08505715037778355, "safety_shutdown": 0.055625594676959024}, "confidence": 0.61752006535862}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 8965, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.251672468999999, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.2518754129999934, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-09", "response": {"answers": {"decision": {"type": "choice", "choice": "require_steward", "probabilities": {"confirm_l3": 0.028200786280120214, "reject_capacity": 0.09423666472844518, "require_steward": 0.8709341900512526, "switch_l4": 0.006628358940182005}, "confidence": 0.8279122534016702}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 8962, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.2972396440000011, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.2974470789999941, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-10", "response": {"answers": {"decision": {"type": "choice", "choice": "reduce_to_30_days", "probabilities": {"approve_45_days": 0.01709624873802973, "deny_sensitive": 0.2705191220020207, "needs_field_review": 0.13669162348278524, "reduce_to_30_days": 0.5756930057771644}, "confidence": 0.4342573410362192}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9168, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3506299899999874, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3508603939999944, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-multi_hop-12", "response": {"answers": {"decision": {"type": "choice", "choice": "needs_authentication", "probabilities": {"covered_recurrence": 0.21622865626371007, "expired": 0.15352634209425553, "needs_authentication": 0.5110559022895967, "wrong_issue_class": 0.11918909935243767}, "confidence": 0.348074536386129}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9016, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.258745177999998, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.259173336999993, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-02", "response": {"answers": {"decision": {"type": "noul", "noul": 0.011348306646321625}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 292, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17429494800001066, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1749196119999965, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-04", "response": {"answers": {"decision": {"type": "noul", "noul": 0.009633436254045717}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 198, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17554439700001012, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.175667408999999, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-05", "response": {"answers": {"decision": {"type": "score", "score": 0.5704623966118885, "probabilities": {"0": 0.721104122846341, "1": 0.07255011809781148, "2": 0.12112499865346583, "3": 0.08522076040238176}, "confidence": 0.42953760338811153, "legend": {"0": "No operative exception.", "1": "Requested but unauthorized exception.", "2": "Authorized, unused exception.", "3": "Authorized exception already used."}}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1236, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6376049929999965, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6377307860000201, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-06", "response": {"answers": {"decision": {"type": "choice", "choice": "keep_open_unresolved", "probabilities": {"await_internal_metrics": 0.004501450007291177, "close_agent_verified": 0.0024759122374671725, "close_confirmed": 0.002725386181242211, "keep_open_unresolved": 0.9902972515739994}, "confidence": 0.9870630020986658}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1192, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6765115000000037, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6767238949999808, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-08", "response": {"answers": {"decision": {"type": "choice", "choice": "late", "probabilities": {"canceled": 0.0026442418135585845, "indeterminate": 0.0016303203384537086, "late": 0.9924659000394311, "on_time": 0.003259537808556577}, "confidence": 0.9899545333859082}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 912, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6719821859999797, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6720961489999979, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-10", "response": {"answers": {"decision": {"type": "choice", "choice": "ready_one_review", "probabilities": {"blocked_failed_test": 0.03803468901717434, "blocked_second_review": 0.3202811353570011, "ready_one_review": 0.4922966804376352, "ready_two_reviews": 0.14938749518818945}, "confidence": 0.32306224058351357}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 629, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.5792426560000195, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.5793543279999938, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-13", "response": {"answers": {"decision": {"type": "noul", "noul": 0.002704399898591743}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.1668361549999986, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16695518699998502, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-a-trap-15", "response": {"answers": {"decision": {"type": "noul", "noul": 0.3487249472273573}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 308, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17641044399999828, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1765313669999955, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-01", "response": {"answers": {"decision": {"type": "noul", "noul": 0.922515831220597}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 288, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16867292800000655, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16877028999999766, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-02", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9155911257070988}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 212, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.1733504430000039, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1734692969999969, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-05", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8709398415967722}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 231, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.15867916899998136, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.15877581099999816, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8358340963624528}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 214, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16690940600000204, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16699924899998564, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-10", "response": {"answers": {"decision": {"type": "noul", "noul": 0.39346645978741673}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 233, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16861999599998967, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16871710800000983, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-14", "response": {"answers": {"decision": {"type": "noul", "noul": 0.01968304239588589}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 250, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16415730400001394, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16425701599999343, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-15", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9391539991694859}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 206, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17089551699999106, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17103435100000297, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-judge_hard-18", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9578978690049913}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 173, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16121110699998553, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1613250690000143, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-long_policy-01", "response": {"answers": {"decision": {"type": "choice", "choice": "approve_lodging_only", "probabilities": {"approve_all": 0.14518066627304826, "approve_lodging_only": 0.3623200730405093, "approve_meals_only": 0.1357719783102017, "reject_entire_claim": 0.35672728237624085}, "confidence": 0.14976009738734572}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 10722, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3705849060000048, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.370793080999988, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-long_policy-02", "response": {"answers": {"decision": {"type": "choice", "choice": "privacy_only", "probabilities": {"analytics_owner_only": 0.15123659105256262, "deny_current_export": 0.17262310163528827, "privacy_and_security": 0.22717405439423885, "privacy_only": 0.44896625291791026}, "confidence": 0.26528833722388034}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 10138, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3544314149999934, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3546260299999915, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-long_policy-05", "response": {"answers": {"decision": {"type": "choice", "choice": "delete_nothing", "probabilities": {"delete_everything": 0.26660026079850574, "delete_nothing": 0.321131116223241, "delete_only_profile": 0.17271097462593896, "delete_profile_omar_april_and_table": 0.23955764835231447}, "confidence": 0.09484148829765456}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9955, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3207738849999942, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3209439379999992, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-long_policy-06", "response": {"answers": {"decision": {"type": "choice", "choice": "east_ward", "probabilities": {"east_ward": 0.6935433681356498, "harbor_neighbors": 0.07620194767545746, "no_award": 0.09957734316680819, "river_arts": 0.13067734102208467}, "confidence": 0.5913911575141997}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 10323, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.4382673959999863, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.438495770000003, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-routing_hard-01", "response": {"answers": {"decision": {"type": "choice", "choice": "office_file_repair", "probabilities": {"data_pipeline": 0.012294358989136361, "desktop_support": 0.014575734496751442, "finance_review": 0.016512150262072818, "office_file_repair": 0.9331846517667616, "spreadsheet_formula": 0.023433104485277714}, "confidence": 0.9164808147084521}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1116, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.794844611000002, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7949561530000153, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-routing_hard-02", "response": {"answers": {"decision": {"type": "choice", "choice": "ledger_reconciliation", "probabilities": {"customer_support": 0.028219969779835763, "fraud_review": 0.0026988764036442237, "ledger_reconciliation": 0.9370539102901865, "mobile_app": 0.016954832306132976, "payment_disputes": 0.015072411220200622}, "confidence": 0.9213173878627331}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1091, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.8066219199999978, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8067326119999905, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-routing_hard-03", "response": {"answers": {"decision": {"type": "choice", "choice": "network_dns", "probabilities": {"api_platform": 0.15280761894496553, "authoritative_dns": 0.05641026224835174, "endpoint_devices": 0.12692377651002615, "network_dns": 0.5953787214163033, "tls_certificates": 0.06847962088035339}, "confidence": 0.4942234017703791}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1089, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7644690810000156, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7645831540000074, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-routing_hard-07", "response": {"answers": {"decision": {"type": "choice", "choice": "code_signing", "probabilities": {"ci_infrastructure": 0.043323107242292824, "code_signing": 0.4327994858191465, "endpoint_security": 0.08112051503736371, "macos_runtime": 0.019608888598070407, "release_notarization": 0.4231480033031266}, "confidence": 0.2909993572739331}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1012, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.7834456020000005, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7835863249999875, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-routing_hard-09", "response": {"answers": {"decision": {"type": "choice", "choice": "source_access", "probabilities": {"git_client": 0.1147722625574645, "identity_login": 0.0059937624549237034, "repository_admin": 0.033855083015527636, "source_access": 0.7941113450531498, "vpn_network": 0.05126754691893432}, "confidence": 0.7426391813164374}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 1030, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 5, "inference_seconds": 0.8056588230000159, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.8057826149999983, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-temporal_numeric-01", "response": {"answers": {"decision": {"type": "choice", "choice": "within_window", "probabilities": {"before_open": 0.12394496077534778, "late_by_2h_or_more": 0.36953660984717124, "late_by_under_2h": 0.029033654873960167, "within_window": 0.47748477450352084}, "confidence": 0.3033130326713611}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 933, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6109319689999779, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6110452029999749, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-temporal_numeric-02", "response": {"answers": {"decision": {"type": "choice", "choice": "over_limit", "probabilities": {"at_least_100gb_below": 0.10796553586798185, "exactly_equal": 0.024349526720124366, "over_limit": 0.8370100213319837, "under_100gb_below": 0.03067491607991008}, "confidence": 0.7826800284426448}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 961, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.6117411930000003, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6118519149999884, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-temporal_numeric-03", "response": {"answers": {"decision": {"type": "choice", "choice": "6_credits", "probabilities": {"12_credits": 0.28869772897781937, "15_credits": 0.13328135031386165, "6_credits": 0.377838863896477, "9_credits": 0.2001820568118421}, "confidence": 0.17045181852863597}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 744, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.614304164999993, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.6144169080000097, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-b-temporal_numeric-04", "response": {"answers": {"decision": {"type": "choice", "choice": "clear_fail", "probabilities": {"clear_fail": 0.5490627768750663, "narrow_fail": 0.445543978379982, "pass": 0.005393244744951596}, "confidence": 0.3235941653125995}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 670, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 3, "inference_seconds": 0.4842478839999842, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.48435564599998315, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-03", "response": {"answers": {"decision": {"type": "noul", "noul": 0.7587143302142999}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 395, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17450142000001279, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1746123730000022, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-05", "response": {"answers": {"decision": {"type": "noul", "noul": 0.364001100781703}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 357, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17137553099999536, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17147778300000027, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-07", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8641695604981507}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 321, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.1625279790000036, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16263705099999015, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-08", "response": {"answers": {"decision": {"type": "noul", "noul": 0.8272500567363228}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 315, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17257727799997724, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1726840910000078, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-09", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9186795312517543}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16420731700000601, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.1643191590000015, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-10", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9432774095699097}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 375, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17265539999999646, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17279518300000518, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-11", "response": {"answers": {"decision": {"type": "noul", "noul": 0.020673753085390826}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 346, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.16962804000002052, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.16976885400001152, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-13", "response": {"answers": {"decision": {"type": "noul", "noul": 0.6502672979372562}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 336, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17153404200001887, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17164090500000384, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-judge_hard-15", "response": {"answers": {"decision": {"type": "noul", "noul": 0.9552272282666985}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 290, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 1, "inference_seconds": 0.17047276799999622, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.17058061100001964, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-multi_hop-07", "response": {"answers": {"decision": {"type": "choice", "choice": "programme_review", "probabilities": {"partial_release": 0.06692633442735359, "programme_review": 0.5208355280324126, "reject_draw": 0.2767749312360586, "release_draw": 0.1354632063041753}, "confidence": 0.3611140373765502}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9956, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.329347557999995, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.3295621929999868, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-multi_hop-09", "response": {"answers": {"decision": {"type": "choice", "choice": "bus_substitution", "probabilities": {"bus_substitution": 0.9008765942907332, "call_fen_cross": 0.006724513766950155, "pass_fen_cross": 0.022150437993719525, "terminate_oakmere": 0.07024845394859715}, "confidence": 0.8678354590543109}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 9999, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.3707928150000157, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.370945679000016, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-multi_hop-11", "response": {"answers": {"decision": {"type": "choice", "choice": "coverage_excluded", "probabilities": {"adjuster_referral": 0.21740838795991788, "coverage_confirmed": 0.13745364825677892, "coverage_excluded": 0.35658002137134026, "documentation_needed": 0.28855794241196303}, "confidence": 0.142106695161787}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 10329, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.4380584449999958, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.4382190999999978, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-multi_hop-12", "response": {"answers": {"decision": {"type": "choice", "choice": "supervised_delivery", "probabilities": {"conservation_hold": 0.23471017573521974, "deny_request": 0.1393813828809644, "digital_copy_only": 0.235065367131667, "supervised_delivery": 0.3908430742521488}, "confidence": 0.18779076566953176}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 2008, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 0.7266347449999842, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 0.7267765279999878, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}
{"id": "benchmark:hard-sol-c-multi_hop-13", "response": {"answers": {"decision": {"type": "choice", "choice": "normal_release", "probabilities": {"engineering_review": 0.04014808586584233, "normal_release": 0.843839139625521, "rework": 0.08389570274111885, "scrap_lot": 0.03211707176751786}, "confidence": 0.7917855195006948}}, "model": "Qwen/Qwen3.5-2B", "usage": {"input_tokens": 10228, "output_tokens": 0}, "metadata": {"method": "lora_decision_head", "temperature": 1.518796342858676, "candidate_sequences": 4, "inference_seconds": 1.4157876929999986, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}, "checkpoint_sha256": "3076462e6356412082e79af909227b39b2863b90def79155ca0821aa506b7ded", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "max_length": 16384, "code_commit": "ed45657bf726c3b77408942830e5578f99df904e"}}, "ok": true, "server_wall_seconds": 1.4159537759999807, "gpu_peak_allocated_bytes": 4427087872, "runtime": {"model": "ZefanCai/Open-Jev-2B", "adapter_revision": "0c7aa498b1627be8da4acf34c863ff0ee0a92785", "base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc", "code_commit": "ed45657bf726c3b77408942830e5578f99df904e", "gpu": "NVIDIA L40S", "dtype": "bfloat16", "batch_size": 1, "max_length": 16384, "prefix_cache": false, "model_load_seconds": 14.586798539000002, "torch_version": "2.8.0+cu128"}}