{"id": "benchmark:original-policy-01-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 81}, "server_wall_seconds": 1.4823472009999996, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.16}}, "usage": {"input_tokens": 81, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.8386788964271545, "true": 0.16132110357284546}}, "native_timing": {"tokens": 81, "state_tokens": 32, "latency_ms": 1481.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-01-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 84}, "server_wall_seconds": 0.08602041600000021, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.13}}, "usage": {"input_tokens": 84, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.8655397891998291, "true": 0.1344602108001709}}, "native_timing": {"tokens": 84, "state_tokens": 35, "latency_ms": 81.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-02-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 70}, "server_wall_seconds": 0.08728460000000027, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.88}}, "usage": {"input_tokens": 70, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.11539839953184128, "true": 0.8846015930175781}}, "native_timing": {"tokens": 70, "state_tokens": 21, "latency_ms": 86.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-02-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 69}, "server_wall_seconds": 0.08481000300000119, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.69}}, "usage": {"input_tokens": 69, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.3147231340408325, "true": 0.6852768063545227}}, "native_timing": {"tokens": 69, "state_tokens": 20, "latency_ms": 83.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-03-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 72}, "server_wall_seconds": 0.08039804500000258, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.56}}, "usage": {"input_tokens": 72, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.44442635774612427, "true": 0.5555736422538757}}, "native_timing": {"tokens": 72, "state_tokens": 23, "latency_ms": 79.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-03-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 71}, "server_wall_seconds": 0.07798203000000115, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.74}}, "usage": {"input_tokens": 71, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.2605655789375305, "true": 0.7394344210624695}}, "native_timing": {"tokens": 71, "state_tokens": 22, "latency_ms": 77.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-04-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 78}, "server_wall_seconds": 0.08236447099999822, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.42}}, "usage": {"input_tokens": 78, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.5838167667388916, "true": 0.4161832332611084}}, "native_timing": {"tokens": 78, "state_tokens": 29, "latency_ms": 81.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-04-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 75}, "server_wall_seconds": 0.07356612499999926, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.13}}, "usage": {"input_tokens": 75, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.8673445582389832, "true": 0.13265548646450043}}, "native_timing": {"tokens": 75, "state_tokens": 26, "latency_ms": 72.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-05-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 82}, "server_wall_seconds": 0.07106741800000194, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.72}}, "usage": {"input_tokens": 82, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.2780306041240692, "true": 0.7219693660736084}}, "native_timing": {"tokens": 82, "state_tokens": 33, "latency_ms": 70.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-05-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.07460416199999997, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.2}}, "usage": {"input_tokens": 74, "output_tokens": 19}}, "unrounded_probabilities": {"decision": {"false": 0.8012946844100952, "true": 0.19870536029338837}}, "native_timing": {"tokens": 74, "state_tokens": 25, "latency_ms": 73.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-06-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 84}, "server_wall_seconds": 0.07112239900000006, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.92}}, "usage": {"input_tokens": 84, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.0833519771695137, "true": 0.9166480302810669}}, "native_timing": {"tokens": 84, "state_tokens": 35, "latency_ms": 70.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-policy-06-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.07731390500000046, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.68}}, "usage": {"input_tokens": 74, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.31776031851768494, "true": 0.6822396516799927}}, "native_timing": {"tokens": 74, "state_tokens": 25, "latency_ms": 76.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-01-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 73}, "server_wall_seconds": 0.08476612500000158, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "refund", "confidence": 0.5, "probabilities": {"cancel": 0.16, "change_address": 0.0, "other": 0.23, "refund": 0.6, "status": 0.0}}}, "usage": {"input_tokens": 73, "output_tokens": 70}}, "unrounded_probabilities": {"decision": {"cancel": 0.16200727224349976, "change_address": 0.004015171900391579, "other": 0.23090803623199463, "refund": 0.6000230312347412, "status": 0.0030464837327599525}}, "native_timing": {"tokens": 73, "state_tokens": 13, "latency_ms": 83.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-01-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.08541707199999848, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.34, "probabilities": {"cancel": 0.13, "change_address": 0.01, "other": 0.47, "refund": 0.39, "status": 0.01}}}, "usage": {"input_tokens": 74, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"cancel": 0.12896940112113953, "change_address": 0.008395141921937466, "other": 0.4711310565471649, "refund": 0.38542652130126953, "status": 0.006077858619391918}}, "native_timing": {"tokens": 74, "state_tokens": 14, "latency_ms": 83.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-02-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.08124850100000103, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.6, "probabilities": {"cancel": 0.03, "change_address": 0.2, "other": 0.68, "refund": 0.01, "status": 0.08}}}, "usage": {"input_tokens": 74, "output_tokens": 72}}, "unrounded_probabilities": {"decision": {"cancel": 0.026980577036738396, "change_address": 0.19582444429397583, "other": 0.6798968315124512, "refund": 0.012799427844583988, "status": 0.08449865877628326}}, "native_timing": {"tokens": 74, "state_tokens": 14, "latency_ms": 80.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-02-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 73}, "server_wall_seconds": 0.08369100599999868, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.31, "probabilities": {"cancel": 0.05, "change_address": 0.07, "other": 0.45, "refund": 0.02, "status": 0.41}}}, "usage": {"input_tokens": 73, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"cancel": 0.04598119109869003, "change_address": 0.07028789818286896, "other": 0.45001232624053955, "refund": 0.022948406636714935, "status": 0.4107701778411865}}, "native_timing": {"tokens": 73, "state_tokens": 13, "latency_ms": 82.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-03-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 75}, "server_wall_seconds": 0.08004560799999894, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "change_address", "confidence": 0.62, "probabilities": {"cancel": 0.02, "change_address": 0.69, "other": 0.25, "refund": 0.01, "status": 0.03}}}, "usage": {"input_tokens": 75, "output_tokens": 75}}, "unrounded_probabilities": {"decision": {"cancel": 0.01677820272743702, "change_address": 0.6937251687049866, "other": 0.24519680440425873, "refund": 0.013280317187309265, "status": 0.031019464135169983}}, "native_timing": {"tokens": 75, "state_tokens": 15, "latency_ms": 79.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-03-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 73}, "server_wall_seconds": 0.08781372700000034, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "change_address", "confidence": 0.77, "probabilities": {"cancel": 0.0, "change_address": 0.81, "other": 0.17, "refund": 0.01, "status": 0.0}}}, "usage": {"input_tokens": 73, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"cancel": 0.004665147978812456, "change_address": 0.8133889436721802, "other": 0.1725817620754242, "refund": 0.006555689033120871, "status": 0.002808394841849804}}, "native_timing": {"tokens": 73, "state_tokens": 13, "latency_ms": 86.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-04-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 78}, "server_wall_seconds": 0.08292432500000047, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.46, "probabilities": {"cancel": 0.32, "change_address": 0.01, "other": 0.57, "refund": 0.1, "status": 0.0}}}, "usage": {"input_tokens": 78, "output_tokens": 72}}, "unrounded_probabilities": {"decision": {"cancel": 0.3155374526977539, "change_address": 0.006760961841791868, "other": 0.5707160234451294, "refund": 0.10226551443338394, "status": 0.004720055963844061}}, "native_timing": {"tokens": 78, "state_tokens": 18, "latency_ms": 81.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-04-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 73}, "server_wall_seconds": 0.07499436299999829, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "cancel", "confidence": 0.66, "probabilities": {"cancel": 0.73, "change_address": 0.0, "other": 0.23, "refund": 0.04, "status": 0.0}}}, "usage": {"input_tokens": 73, "output_tokens": 72}}, "unrounded_probabilities": {"decision": {"cancel": 0.7299904227256775, "change_address": 0.0027535089757293463, "other": 0.2289748340845108, "refund": 0.03613849729299545, "status": 0.002142836106941104}}, "native_timing": {"tokens": 73, "state_tokens": 13, "latency_ms": 74.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-05-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 73}, "server_wall_seconds": 0.07415018000000018, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.64, "probabilities": {"cancel": 0.04, "change_address": 0.01, "other": 0.71, "refund": 0.23, "status": 0.01}}}, "usage": {"input_tokens": 73, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"cancel": 0.03648090735077858, "change_address": 0.010190865956246853, "other": 0.7109861373901367, "refund": 0.23209720849990845, "status": 0.010244914330542088}}, "native_timing": {"tokens": 73, "state_tokens": 13, "latency_ms": 73.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-05-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 72}, "server_wall_seconds": 0.07506649400000143, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "refund", "confidence": 0.38, "probabilities": {"cancel": 0.04, "change_address": 0.01, "other": 0.45, "refund": 0.5, "status": 0.01}}}, "usage": {"input_tokens": 72, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"cancel": 0.03520656004548073, "change_address": 0.008726131170988083, "other": 0.44605711102485657, "refund": 0.5019126534461975, "status": 0.00809753593057394}}, "native_timing": {"tokens": 72, "state_tokens": 12, "latency_ms": 74.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-06-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 71}, "server_wall_seconds": 0.07514803599999809, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.19, "probabilities": {"cancel": 0.23, "change_address": 0.05, "other": 0.35, "refund": 0.07, "status": 0.3}}}, "usage": {"input_tokens": 71, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"cancel": 0.2294117659330368, "change_address": 0.048765357583761215, "other": 0.3498203754425049, "refund": 0.07457809895277023, "status": 0.29742440581321716}}, "native_timing": {"tokens": 71, "state_tokens": 11, "latency_ms": 74.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-intent-06-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 76}, "server_wall_seconds": 0.08297440599999817, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "other", "confidence": 0.3, "probabilities": {"cancel": 0.3, "change_address": 0.03, "other": 0.44, "refund": 0.1, "status": 0.13}}}, "usage": {"input_tokens": 76, "output_tokens": 71}}, "unrounded_probabilities": {"decision": {"cancel": 0.30450576543807983, "change_address": 0.026209641247987747, "other": 0.4425531327724457, "refund": 0.09620532393455505, "status": 0.13052614033222198}}, "native_timing": {"tokens": 76, "state_tokens": 16, "latency_ms": 82.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-01-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 72}, "server_wall_seconds": 0.07747447899999926, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 0.36, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.78, "1": 0.11, "2": 0.07, "3": 0.04}, "confidence": 0.88}}, "usage": {"input_tokens": 72, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.7840684056282043, "1": 0.10919222235679626, "2": 0.06933045387268066, "3": 0.03740894794464111}}, "native_timing": {"tokens": 72, "state_tokens": 11, "latency_ms": 76.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-01-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.08055001199999978, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 0.13, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.92, "1": 0.05, "2": 0.02, "3": 0.01}, "confidence": 0.96}}, "usage": {"input_tokens": 74, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.9188807010650635, "1": 0.04504450410604477, "2": 0.02302812598645687, "3": 0.013046737760305405}}, "native_timing": {"tokens": 74, "state_tokens": 13, "latency_ms": 79.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-02-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 76}, "server_wall_seconds": 0.07908029200000044, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 0.96, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.32, "1": 0.46, "2": 0.16, "3": 0.06}, "confidence": 0.8}}, "usage": {"input_tokens": 76, "output_tokens": 125}}, "unrounded_probabilities": {"decision": {"0": 0.3161374032497406, "1": 0.46453797817230225, "2": 0.1618575006723404, "3": 0.05746708810329437}}, "native_timing": {"tokens": 76, "state_tokens": 15, "latency_ms": 77.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-02-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 77}, "server_wall_seconds": 0.07863433900000061, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 0.93, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.27, "1": 0.59, "2": 0.09, "3": 0.06}, "confidence": 0.84}}, "usage": {"input_tokens": 77, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.27021893858909607, "1": 0.5866365432739258, "2": 0.08681365102529526, "3": 0.0563308410346508}}, "native_timing": {"tokens": 77, "state_tokens": 16, "latency_ms": 77.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-03-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 73}, "server_wall_seconds": 0.07685320200000234, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.63, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.16, "1": 0.14, "2": 0.62, "3": 0.08}, "confidence": 0.82}}, "usage": {"input_tokens": 73, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.15541385114192963, "1": 0.1449718475341797, "2": 0.6152814030647278, "3": 0.0843329131603241}}, "native_timing": {"tokens": 73, "state_tokens": 12, "latency_ms": 75.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-03-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 75}, "server_wall_seconds": 0.07819575799999967, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.53, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.22, "1": 0.09, "2": 0.62, "3": 0.07}, "confidence": 0.8}}, "usage": {"input_tokens": 75, "output_tokens": 125}}, "unrounded_probabilities": {"decision": {"0": 0.2243546098470688, "1": 0.09127053618431091, "2": 0.6178698539733887, "3": 0.06650491803884506}}, "native_timing": {"tokens": 75, "state_tokens": 14, "latency_ms": 77.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-04-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 75}, "server_wall_seconds": 0.07918724399999988, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 2.79, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.05, "1": 0.01, "2": 0.04, "3": 0.9}, "confidence": 0.93}}, "usage": {"input_tokens": 75, "output_tokens": 125}}, "unrounded_probabilities": {"decision": {"0": 0.04961699992418289, "1": 0.013822833076119423, "2": 0.03531978651881218, "3": 0.901240348815918}}, "native_timing": {"tokens": 75, "state_tokens": 14, "latency_ms": 78.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-04-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.07891490699999792, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 2.74, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.05, "1": 0.02, "2": 0.06, "3": 0.87}, "confidence": 0.91}}, "usage": {"input_tokens": 74, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.053166210651397705, "1": 0.02075055055320263, "2": 0.060374241322278976, "3": 0.8657090067863464}}, "native_timing": {"tokens": 74, "state_tokens": 13, "latency_ms": 77.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-05-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 78}, "server_wall_seconds": 0.08430316200000121, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.34, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.33, "1": 0.08, "2": 0.51, "3": 0.08}, "confidence": 0.72}}, "usage": {"input_tokens": 78, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.3345189690589905, "1": 0.07643093168735504, "2": 0.5056872963905334, "3": 0.08336280286312103}}, "native_timing": {"tokens": 78, "state_tokens": 17, "latency_ms": 83.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-05-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 77}, "server_wall_seconds": 0.07511107599999889, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.05, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.45, "1": 0.1, "2": 0.4, "3": 0.05}, "confidence": 0.65}}, "usage": {"input_tokens": 77, "output_tokens": 124}}, "unrounded_probabilities": {"decision": {"0": 0.4546709358692169, "1": 0.09514999389648438, "2": 0.3975710868835449, "3": 0.052607983350753784}}, "native_timing": {"tokens": 77, "state_tokens": 16, "latency_ms": 74.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-06-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 78}, "server_wall_seconds": 0.07452933899999792, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.45, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.23, "1": 0.14, "2": 0.6, "3": 0.03}, "confidence": 0.79}}, "usage": {"input_tokens": 78, "output_tokens": 125}}, "unrounded_probabilities": {"decision": {"0": 0.22660908102989197, "1": 0.13620445132255554, "2": 0.602436363697052, "3": 0.034750089049339294}}, "native_timing": {"tokens": 78, "state_tokens": 17, "latency_ms": 73.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-ordinal-06-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 85}, "server_wall_seconds": 0.07777126600000273, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.38, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.25, "1": 0.17, "2": 0.52, "3": 0.06}, "confidence": 0.76}}, "usage": {"input_tokens": 85, "output_tokens": 126}}, "unrounded_probabilities": {"decision": {"0": 0.25479137897491455, "1": 0.1654099076986313, "2": 0.5226662755012512, "3": 0.05713238939642906}}, "native_timing": {"tokens": 85, "state_tokens": 24, "latency_ms": 76.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-01-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 64}, "server_wall_seconds": 0.10111554200000228, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "unknown", "confidence": 0.22, "probabilities": {"courier": 0.08, "pickup": 0.37, "post": 0.13, "unknown": 0.42}}}, "usage": {"input_tokens": 64, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.08198559284210205, "pickup": 0.3700275421142578, "post": 0.13041305541992188, "unknown": 0.41757383942604065}}, "native_timing": {"tokens": 64, "state_tokens": 12, "latency_ms": 100.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-01-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 66}, "server_wall_seconds": 0.07512208500000028, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "pickup", "confidence": 0.3, "probabilities": {"courier": 0.05, "pickup": 0.47, "post": 0.07, "unknown": 0.41}}}, "usage": {"input_tokens": 66, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"courier": 0.046795204281806946, "pickup": 0.4719495475292206, "post": 0.06941290944814682, "unknown": 0.41184237599372864}}, "native_timing": {"tokens": 66, "state_tokens": 14, "latency_ms": 74.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-02-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.07380340000000274, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "unknown", "confidence": 0.77, "probabilities": {"courier": 0.03, "pickup": 0.04, "post": 0.1, "unknown": 0.83}}}, "usage": {"input_tokens": 67, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"courier": 0.03479795530438423, "pickup": 0.04226212576031685, "post": 0.09703127294778824, "unknown": 0.8259086608886719}}, "native_timing": {"tokens": 67, "state_tokens": 15, "latency_ms": 72.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-02-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 66}, "server_wall_seconds": 0.08024118299999827, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "unknown", "confidence": 0.78, "probabilities": {"courier": 0.02, "pickup": 0.03, "post": 0.11, "unknown": 0.84}}}, "usage": {"input_tokens": 66, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.02423354797065258, "pickup": 0.025219174101948738, "post": 0.1120418831706047, "unknown": 0.8385053873062134}}, "native_timing": {"tokens": 66, "state_tokens": 14, "latency_ms": 79.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-03-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 70}, "server_wall_seconds": 0.07882480399999992, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "unknown", "confidence": 0.25, "probabilities": {"courier": 0.29, "pickup": 0.11, "post": 0.16, "unknown": 0.44}}}, "usage": {"input_tokens": 70, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.2940341830253601, "pickup": 0.11385902017354965, "post": 0.1568729430437088, "unknown": 0.43523386120796204}}, "native_timing": {"tokens": 70, "state_tokens": 18, "latency_ms": 77.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-03-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 69}, "server_wall_seconds": 0.0802272429999995, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "courier", "confidence": 0.73, "probabilities": {"courier": 0.79, "pickup": 0.01, "post": 0.1, "unknown": 0.09}}}, "usage": {"input_tokens": 69, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.7938085198402405, "pickup": 0.00736364396288991, "post": 0.10463446378707886, "unknown": 0.09419335424900055}}, "native_timing": {"tokens": 69, "state_tokens": 17, "latency_ms": 79.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-04-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.07628919600000117, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "post", "confidence": 0.66, "probabilities": {"courier": 0.03, "pickup": 0.02, "post": 0.74, "unknown": 0.2}}}, "usage": {"input_tokens": 67, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"courier": 0.030109843239188194, "pickup": 0.023950597271323204, "post": 0.7432087063789368, "unknown": 0.20273083448410034}}, "native_timing": {"tokens": 67, "state_tokens": 15, "latency_ms": 75.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-04-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 64}, "server_wall_seconds": 0.07067859600000048, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "post", "confidence": 0.76, "probabilities": {"courier": 0.02, "pickup": 0.01, "post": 0.82, "unknown": 0.15}}}, "usage": {"input_tokens": 64, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.024639897048473358, "pickup": 0.01260473020374775, "post": 0.8174852132797241, "unknown": 0.14527012407779694}}, "native_timing": {"tokens": 64, "state_tokens": 12, "latency_ms": 69.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-05-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.08145653500000094, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "unknown", "confidence": 0.91, "probabilities": {"courier": 0.02, "pickup": 0.03, "post": 0.02, "unknown": 0.93}}}, "usage": {"input_tokens": 67, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.022271210327744484, "pickup": 0.02701045572757721, "post": 0.019079728052020073, "unknown": 0.9316385984420776}}, "native_timing": {"tokens": 67, "state_tokens": 15, "latency_ms": 80.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-05-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 63}, "server_wall_seconds": 0.07506048400000154, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "unknown", "confidence": 0.75, "probabilities": {"courier": 0.09, "pickup": 0.04, "post": 0.06, "unknown": 0.81}}}, "usage": {"input_tokens": 63, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.0866222158074379, "pickup": 0.03859835863113403, "post": 0.06279172748327255, "unknown": 0.8119876384735107}}, "native_timing": {"tokens": 63, "state_tokens": 11, "latency_ms": 74.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-06-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.07570632000000188, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "courier", "confidence": 0.66, "probabilities": {"courier": 0.74, "pickup": 0.03, "post": 0.06, "unknown": 0.17}}}, "usage": {"input_tokens": 67, "output_tokens": 66}}, "unrounded_probabilities": {"decision": {"courier": 0.7431442737579346, "pickup": 0.02791612036526203, "post": 0.06311632692813873, "unknown": 0.16582326591014862}}, "native_timing": {"tokens": 67, "state_tokens": 15, "latency_ms": 74.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-extraction-06-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 66}, "server_wall_seconds": 0.0787370220000021, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "courier", "confidence": 0.7, "probabilities": {"courier": 0.77, "pickup": 0.03, "post": 0.02, "unknown": 0.18}}}, "usage": {"input_tokens": 66, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"courier": 0.7734956741333008, "pickup": 0.027482343837618828, "post": 0.020735500380396843, "unknown": 0.17828643321990967}}, "native_timing": {"tokens": 66, "state_tokens": 14, "latency_ms": 77.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-01-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 63}, "server_wall_seconds": 0.07418750000000074, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.95}}, "usage": {"input_tokens": 63, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.05173182860016823, "true": 0.9482681751251221}}, "native_timing": {"tokens": 63, "state_tokens": 20, "latency_ms": 73.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-01-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 59}, "server_wall_seconds": 0.07498189100000019, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.6}}, "usage": {"input_tokens": 59, "output_tokens": 19}}, "unrounded_probabilities": {"decision": {"false": 0.4025387465953827, "true": 0.5974612236022949}}, "native_timing": {"tokens": 59, "state_tokens": 16, "latency_ms": 74.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-02-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 60}, "server_wall_seconds": 0.0789645479999983, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.95}}, "usage": {"input_tokens": 60, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.04744277521967888, "true": 0.9525572657585144}}, "native_timing": {"tokens": 60, "state_tokens": 17, "latency_ms": 77.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-02-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 61}, "server_wall_seconds": 0.0753896819999973, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.06}}, "usage": {"input_tokens": 61, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.9393520355224609, "true": 0.06064799800515175}}, "native_timing": {"tokens": 61, "state_tokens": 18, "latency_ms": 74.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-03-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 63}, "server_wall_seconds": 0.075966477999998, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.98}}, "usage": {"input_tokens": 63, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.021280359476804733, "true": 0.9787196516990662}}, "native_timing": {"tokens": 63, "state_tokens": 20, "latency_ms": 75.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-03-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.07809913499999865, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.9}}, "usage": {"input_tokens": 67, "output_tokens": 19}}, "unrounded_probabilities": {"decision": {"false": 0.1001611277461052, "true": 0.8998388051986694}}, "native_timing": {"tokens": 67, "state_tokens": 24, "latency_ms": 77.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-04-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.07710895800000017, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.23}}, "usage": {"input_tokens": 67, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.7721205949783325, "true": 0.2278793752193451}}, "native_timing": {"tokens": 67, "state_tokens": 24, "latency_ms": 76.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-04-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 64}, "server_wall_seconds": 0.07813764599999828, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.47}}, "usage": {"input_tokens": 64, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.5336866974830627, "true": 0.46631327271461487}}, "native_timing": {"tokens": 64, "state_tokens": 21, "latency_ms": 77.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-05-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 56}, "server_wall_seconds": 0.08235253899999861, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.63}}, "usage": {"input_tokens": 56, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.3727853298187256, "true": 0.6272147297859192}}, "native_timing": {"tokens": 56, "state_tokens": 13, "latency_ms": 81.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-05-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 57}, "server_wall_seconds": 0.07233404000000121, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.82}}, "usage": {"input_tokens": 57, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.18424257636070251, "true": 0.8157573938369751}}, "native_timing": {"tokens": 57, "state_tokens": 14, "latency_ms": 71.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-06-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 72}, "server_wall_seconds": 0.07172450399999875, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.86}}, "usage": {"input_tokens": 72, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.14164882898330688, "true": 0.8583512306213379}}, "native_timing": {"tokens": 72, "state_tokens": 29, "latency_ms": 71.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-adequacy-06-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.08097798200000028, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.91}}, "usage": {"input_tokens": 74, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.09352824091911316, "true": 0.9064717888832092}}, "native_timing": {"tokens": 74, "state_tokens": 31, "latency_ms": 80.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-01-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 102}, "server_wall_seconds": 0.10997358999999918, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "math", "confidence": 0.76, "probabilities": {"coding": 0.07, "coding_agent": 0.01, "document": 0.04, "general": 0.04, "math": 0.8, "tools": 0.03}}}, "usage": {"input_tokens": 102, "output_tokens": 82}}, "unrounded_probabilities": {"decision": {"coding": 0.07309307903051376, "coding_agent": 0.009065038524568081, "document": 0.041633084416389465, "general": 0.04191281646490097, "math": 0.8026856780052185, "tools": 0.03161029517650604}}, "native_timing": {"tokens": 102, "state_tokens": 15, "latency_ms": 108.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-01-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 98}, "server_wall_seconds": 0.07954474299999958, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "math", "confidence": 0.76, "probabilities": {"coding": 0.05, "coding_agent": 0.01, "document": 0.05, "general": 0.05, "math": 0.8, "tools": 0.04}}}, "usage": {"input_tokens": 98, "output_tokens": 82}}, "unrounded_probabilities": {"decision": {"coding": 0.04931257292628288, "coding_agent": 0.010179863311350346, "document": 0.05016465485095978, "general": 0.04788198694586754, "math": 0.8037091493606567, "tools": 0.03875180333852768}}, "native_timing": {"tokens": 98, "state_tokens": 11, "latency_ms": 78.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-02-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 103}, "server_wall_seconds": 0.08188692599999925, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "coding", "confidence": 0.85, "probabilities": {"coding": 0.88, "coding_agent": 0.01, "document": 0.01, "general": 0.08, "math": 0.01, "tools": 0.01}}}, "usage": {"input_tokens": 103, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.8789894580841064, "coding_agent": 0.013816106133162975, "document": 0.013437454588711262, "general": 0.0767718181014061, "math": 0.01022972259670496, "tools": 0.006755397189408541}}, "native_timing": {"tokens": 103, "state_tokens": 16, "latency_ms": 80.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-02-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 97}, "server_wall_seconds": 0.07745322699999946, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "coding", "confidence": 0.76, "probabilities": {"coding": 0.8, "coding_agent": 0.06, "document": 0.02, "general": 0.1, "math": 0.01, "tools": 0.01}}}, "usage": {"input_tokens": 97, "output_tokens": 81}}, "unrounded_probabilities": {"decision": {"coding": 0.7991024255752563, "coding_agent": 0.05655611306428909, "document": 0.02263730764389038, "general": 0.10245455801486969, "math": 0.0076329815201461315, "tools": 0.011616529896855354}}, "native_timing": {"tokens": 97, "state_tokens": 10, "latency_ms": 76.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-03-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 102}, "server_wall_seconds": 0.07403540600000014, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "coding_agent", "confidence": 0.75, "probabilities": {"coding": 0.08, "coding_agent": 0.79, "document": 0.02, "general": 0.08, "math": 0.0, "tools": 0.03}}}, "usage": {"input_tokens": 102, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.07585307955741882, "coding_agent": 0.7937270998954773, "document": 0.01745128631591797, "general": 0.07603095471858978, "math": 0.003362160176038742, "tools": 0.033575400710105896}}, "native_timing": {"tokens": 102, "state_tokens": 15, "latency_ms": 73.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-03-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 104}, "server_wall_seconds": 0.07837810299999859, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "coding_agent", "confidence": 0.78, "probabilities": {"coding": 0.04, "coding_agent": 0.82, "document": 0.03, "general": 0.07, "math": 0.0, "tools": 0.04}}}, "usage": {"input_tokens": 104, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.03983987122774124, "coding_agent": 0.8151764273643494, "document": 0.026542261242866516, "general": 0.07460379600524902, "math": 0.0029313426930457354, "tools": 0.04090630263090134}}, "native_timing": {"tokens": 104, "state_tokens": 17, "latency_ms": 77.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-04-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 98}, "server_wall_seconds": 0.08326236399999942, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "document", "confidence": 0.69, "probabilities": {"coding": 0.03, "coding_agent": 0.03, "document": 0.74, "general": 0.13, "math": 0.02, "tools": 0.05}}}, "usage": {"input_tokens": 98, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.032438602298498154, "coding_agent": 0.030191857367753983, "document": 0.7420311570167542, "general": 0.13277338445186615, "math": 0.015928808599710464, "tools": 0.04663623124361038}}, "native_timing": {"tokens": 98, "state_tokens": 11, "latency_ms": 82.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-04-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 97}, "server_wall_seconds": 0.08238153000000281, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "document", "confidence": 0.6, "probabilities": {"coding": 0.06, "coding_agent": 0.06, "document": 0.67, "general": 0.09, "math": 0.05, "tools": 0.08}}}, "usage": {"input_tokens": 97, "output_tokens": 82}}, "unrounded_probabilities": {"decision": {"coding": 0.05897926911711693, "coding_agent": 0.058390188962221146, "document": 0.665703535079956, "general": 0.0862223282456398, "math": 0.04653475433588028, "tools": 0.08416996151208878}}, "native_timing": {"tokens": 97, "state_tokens": 10, "latency_ms": 81.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-05-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 99}, "server_wall_seconds": 0.0739873650000007, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "tools", "confidence": 0.79, "probabilities": {"coding": 0.02, "coding_agent": 0.01, "document": 0.04, "general": 0.1, "math": 0.0, "tools": 0.83}}}, "usage": {"input_tokens": 99, "output_tokens": 81}}, "unrounded_probabilities": {"decision": {"coding": 0.023492395877838135, "coding_agent": 0.007391664199531078, "document": 0.04108307510614395, "general": 0.09880248457193375, "math": 0.003832418005913496, "tools": 0.8253979682922363}}, "native_timing": {"tokens": 99, "state_tokens": 12, "latency_ms": 73.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-05-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 99}, "server_wall_seconds": 0.0753779630000011, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "tools", "confidence": 0.26, "probabilities": {"coding": 0.05, "coding_agent": 0.03, "document": 0.15, "general": 0.37, "math": 0.01, "tools": 0.38}}}, "usage": {"input_tokens": 99, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.04804099723696709, "coding_agent": 0.02849528379738331, "document": 0.15460330247879028, "general": 0.3730098009109497, "math": 0.013019077479839325, "tools": 0.38283154368400574}}, "native_timing": {"tokens": 99, "state_tokens": 12, "latency_ms": 74.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-06-0", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 98}, "server_wall_seconds": 0.07385787199999783, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "general", "confidence": 0.67, "probabilities": {"coding": 0.05, "coding_agent": 0.03, "document": 0.12, "general": 0.72, "math": 0.01, "tools": 0.06}}}, "usage": {"input_tokens": 98, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.0529228076338768, "coding_agent": 0.03143659234046936, "document": 0.11610125750303268, "general": 0.7231309413909912, "math": 0.012832600623369217, "tools": 0.06357578933238983}}, "native_timing": {"tokens": 98, "state_tokens": 11, "latency_ms": 73.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:original-routing-06-1", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 97}, "server_wall_seconds": 0.07932575799999952, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "general", "confidence": 0.74, "probabilities": {"coding": 0.05, "coding_agent": 0.03, "document": 0.08, "general": 0.78, "math": 0.01, "tools": 0.05}}}, "usage": {"input_tokens": 97, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"coding": 0.04794248566031456, "coding_agent": 0.026316598057746887, "document": 0.07950342446565628, "general": 0.7823469638824463, "math": 0.011729603633284569, "tools": 0.05216098949313164}}, "native_timing": {"tokens": 97, "state_tokens": 10, "latency_ms": 78.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-00", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 93}, "server_wall_seconds": 0.09852977399999929, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "track_order", "confidence": 0.97, "probabilities": {"billing_question": 0.0, "cancel_order": 0.01, "change_address": 0.0, "report_damage": 0.01, "track_order": 0.97}}}, "usage": {"input_tokens": 93, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"billing_question": 0.002641286002472043, "cancel_order": 0.011788918636739254, "change_address": 0.004078633617609739, "report_damage": 0.007148850709199905, "track_order": 0.9743423461914062}}, "native_timing": {"tokens": 93, "state_tokens": 18, "latency_ms": 97.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 93}, "server_wall_seconds": 0.07734573500000153, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "cancel_order", "confidence": 0.99, "probabilities": {"billing_question": 0.0, "cancel_order": 0.99, "change_address": 0.0, "report_damage": 0.01, "track_order": 0.0}}}, "usage": {"input_tokens": 93, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"billing_question": 0.0011598406126722693, "cancel_order": 0.9893296360969543, "change_address": 0.0009986282093450427, "report_damage": 0.007393687963485718, "track_order": 0.0011181439040228724}}, "native_timing": {"tokens": 93, "state_tokens": 18, "latency_ms": 76.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 96}, "server_wall_seconds": 0.0883351800000014, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "change_address", "confidence": 0.98, "probabilities": {"billing_question": 0.0, "cancel_order": 0.01, "change_address": 0.99, "report_damage": 0.0, "track_order": 0.0}}}, "usage": {"input_tokens": 96, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"billing_question": 0.0008558141998946667, "cancel_order": 0.007204018998891115, "change_address": 0.987266480922699, "report_damage": 0.0020023686811327934, "track_order": 0.0026712343096733093}}, "native_timing": {"tokens": 96, "state_tokens": 21, "latency_ms": 86.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 85}, "server_wall_seconds": 0.0829249350000012, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "report_damage", "confidence": 0.97, "probabilities": {"billing_question": 0.0, "cancel_order": 0.01, "change_address": 0.0, "report_damage": 0.98, "track_order": 0.01}}}, "usage": {"input_tokens": 85, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"billing_question": 0.001163716078735888, "cancel_order": 0.014234941452741623, "change_address": 0.0010818245355039835, "report_damage": 0.9758220314979553, "track_order": 0.007697509601712227}}, "native_timing": {"tokens": 85, "state_tokens": 10, "latency_ms": 81.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 90}, "server_wall_seconds": 0.07953060399999856, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "billing_question", "confidence": 0.85, "probabilities": {"billing_question": 0.88, "cancel_order": 0.07, "change_address": 0.01, "report_damage": 0.03, "track_order": 0.01}}}, "usage": {"input_tokens": 90, "output_tokens": 79}}, "unrounded_probabilities": {"decision": {"billing_question": 0.8763135671615601, "cancel_order": 0.06544576585292816, "change_address": 0.011465511284768581, "report_damage": 0.031992409378290176, "track_order": 0.014782700687646866}}, "native_timing": {"tokens": 90, "state_tokens": 15, "latency_ms": 78.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 89}, "server_wall_seconds": 0.07623196599999815, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "track_order", "confidence": 0.71, "probabilities": {"billing_question": 0.03, "cancel_order": 0.12, "change_address": 0.03, "report_damage": 0.06, "track_order": 0.77}}}, "usage": {"input_tokens": 89, "output_tokens": 79}}, "unrounded_probabilities": {"decision": {"billing_question": 0.030802372843027115, "cancel_order": 0.11834641546010971, "change_address": 0.028192240744829178, "report_damage": 0.05765002593398094, "track_order": 0.7650090456008911}}, "native_timing": {"tokens": 89, "state_tokens": 14, "latency_ms": 75.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 81}, "server_wall_seconds": 0.07990993199999963, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "set_alarm", "confidence": 0.8, "probabilities": {"play_music": 0.0, "send_message": 0.15, "set_alarm": 0.84, "turn_off_lights": 0.0, "weather": 0.0}}}, "usage": {"input_tokens": 81, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"play_music": 0.0005431263707578182, "send_message": 0.15443047881126404, "set_alarm": 0.8421052694320679, "turn_off_lights": 0.0011603791499510407, "weather": 0.0017607645131647587}}, "native_timing": {"tokens": 81, "state_tokens": 13, "latency_ms": 78.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 74}, "server_wall_seconds": 0.08178989299999984, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "play_music", "confidence": 0.99, "probabilities": {"play_music": 0.99, "send_message": 0.0, "set_alarm": 0.0, "turn_off_lights": 0.0, "weather": 0.0}}}, "usage": {"input_tokens": 74, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"play_music": 0.9911760091781616, "send_message": 0.0033059949055314064, "set_alarm": 0.002048364607617259, "turn_off_lights": 0.002503775293007493, "weather": 0.0009658037452027202}}, "native_timing": {"tokens": 74, "state_tokens": 6, "latency_ms": 80.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 76}, "server_wall_seconds": 0.07119278900000126, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "weather", "confidence": 0.99, "probabilities": {"play_music": 0.0, "send_message": 0.0, "set_alarm": 0.0, "turn_off_lights": 0.0, "weather": 0.99}}}, "usage": {"input_tokens": 76, "output_tokens": 75}}, "unrounded_probabilities": {"decision": {"play_music": 0.0007339987205341458, "send_message": 0.004242720082402229, "set_alarm": 0.0007615720969624817, "turn_off_lights": 0.0007223968859761953, "weather": 0.9935393929481506}}, "native_timing": {"tokens": 76, "state_tokens": 8, "latency_ms": 70.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 79}, "server_wall_seconds": 0.0691732949999988, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "send_message", "confidence": 0.89, "probabilities": {"play_music": 0.01, "send_message": 0.91, "set_alarm": 0.06, "turn_off_lights": 0.01, "weather": 0.02}}}, "usage": {"input_tokens": 79, "output_tokens": 80}}, "unrounded_probabilities": {"decision": {"play_music": 0.009079476818442345, "send_message": 0.9101366996765137, "set_alarm": 0.05612970516085625, "turn_off_lights": 0.00888892263174057, "weather": 0.015765182673931122}}, "native_timing": {"tokens": 79, "state_tokens": 11, "latency_ms": 68.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 77}, "server_wall_seconds": 0.07028200400000273, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "turn_off_lights", "confidence": 0.99, "probabilities": {"play_music": 0.0, "send_message": 0.0, "set_alarm": 0.0, "turn_off_lights": 0.99, "weather": 0.0}}}, "usage": {"input_tokens": 77, "output_tokens": 78}}, "unrounded_probabilities": {"decision": {"play_music": 0.0009155551088042557, "send_message": 0.0027404159773141146, "set_alarm": 0.0019460574258118868, "turn_off_lights": 0.993006706237793, "weather": 0.0013912779977545142}}, "native_timing": {"tokens": 77, "state_tokens": 9, "latency_ms": 69.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-intent-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 77}, "server_wall_seconds": 0.07279197099999735, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "set_alarm", "confidence": 0.99, "probabilities": {"play_music": 0.0, "send_message": 0.0, "set_alarm": 0.99, "turn_off_lights": 0.0, "weather": 0.0}}}, "usage": {"input_tokens": 77, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"play_music": 0.0009037060663104057, "send_message": 0.002625074703246355, "set_alarm": 0.9931334853172302, "turn_off_lights": 0.002276445273309946, "weather": 0.001061334740370512}}, "native_timing": {"tokens": 77, "state_tokens": 9, "latency_ms": 71.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-00", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 61}, "server_wall_seconds": 0.07262726699999789, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.97}}, "usage": {"input_tokens": 61, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.034283772110939026, "true": 0.9657162427902222}}, "native_timing": {"tokens": 61, "state_tokens": 20, "latency_ms": 71.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 59}, "server_wall_seconds": 0.07255594600000137, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.02}}, "usage": {"input_tokens": 59, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.9762128591537476, "true": 0.023787103593349457}}, "native_timing": {"tokens": 59, "state_tokens": 18, "latency_ms": 71.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 67}, "server_wall_seconds": 0.08306532699999991, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.98}}, "usage": {"input_tokens": 67, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.01584840938448906, "true": 0.9841516017913818}}, "native_timing": {"tokens": 67, "state_tokens": 27, "latency_ms": 82.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 70}, "server_wall_seconds": 0.085832620999998, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.01}}, "usage": {"input_tokens": 70, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.9896223545074463, "true": 0.010377678088843822}}, "native_timing": {"tokens": 70, "state_tokens": 30, "latency_ms": 84.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 59}, "server_wall_seconds": 0.07748980799999927, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.95}}, "usage": {"input_tokens": 59, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.049203962087631226, "true": 0.9507960081100464}}, "native_timing": {"tokens": 59, "state_tokens": 17, "latency_ms": 76.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 60}, "server_wall_seconds": 0.08412860600000016, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.13}}, "usage": {"input_tokens": 60, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.8709396719932556, "true": 0.1290602684020996}}, "native_timing": {"tokens": 60, "state_tokens": 18, "latency_ms": 82.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 80}, "server_wall_seconds": 0.08973396699999725, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.98}}, "usage": {"input_tokens": 80, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.01827847585082054, "true": 0.9817215204238892}}, "native_timing": {"tokens": 80, "state_tokens": 25, "latency_ms": 89.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 55}, "server_wall_seconds": 0.06837886499999968, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.03}}, "usage": {"input_tokens": 55, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.9709040522575378, "true": 0.029095908626914024}}, "native_timing": {"tokens": 55, "state_tokens": 12, "latency_ms": 67.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 51}, "server_wall_seconds": 0.06466395499999678, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.99}}, "usage": {"input_tokens": 51, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.011997099965810776, "true": 0.9880028963088989}}, "native_timing": {"tokens": 51, "state_tokens": 10, "latency_ms": 64.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 51}, "server_wall_seconds": 0.07241322399999817, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.01}}, "usage": {"input_tokens": 51, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.9884800910949707, "true": 0.011519882827997208}}, "native_timing": {"tokens": 51, "state_tokens": 10, "latency_ms": 71.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 60}, "server_wall_seconds": 0.06577097500000306, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.99}}, "usage": {"input_tokens": 60, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.010138199664652348, "true": 0.9898618459701538}}, "native_timing": {"tokens": 60, "state_tokens": 15, "latency_ms": 65.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-fact-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 61}, "server_wall_seconds": 0.06499259499999965, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.02}}, "usage": {"input_tokens": 61, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.978111982345581, "true": 0.02188800275325775}}, "native_timing": {"tokens": 61, "state_tokens": 16, "latency_ms": 64.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-00", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 50}, "server_wall_seconds": 0.07539348299999915, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "paypal", "confidence": 0.97, "probabilities": {"bank_transfer": 0.01, "cash": 0.0, "credit_card": 0.01, "paypal": 0.98}}}, "usage": {"input_tokens": 50, "output_tokens": 65}}, "unrounded_probabilities": {"decision": {"bank_transfer": 0.005705555435270071, "cash": 0.0029326959047466516, "credit_card": 0.010485039092600346, "paypal": 0.9808767437934875}}, "native_timing": {"tokens": 50, "state_tokens": 8, "latency_ms": 74.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 52}, "server_wall_seconds": 0.07563612999999947, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "cash", "confidence": 0.94, "probabilities": {"bank_transfer": 0.01, "cash": 0.96, "credit_card": 0.03, "paypal": 0.01}}}, "usage": {"input_tokens": 52, "output_tokens": 66}}, "unrounded_probabilities": {"decision": {"bank_transfer": 0.010208235122263432, "cash": 0.958379328250885, "credit_card": 0.025033866986632347, "paypal": 0.0063785226084291935}}, "native_timing": {"tokens": 52, "state_tokens": 10, "latency_ms": 74.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 53}, "server_wall_seconds": 0.0728204450000014, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "bank_transfer", "confidence": 0.95, "probabilities": {"bank_transfer": 0.97, "cash": 0.01, "credit_card": 0.02, "paypal": 0.01}}}, "usage": {"input_tokens": 53, "output_tokens": 67}}, "unrounded_probabilities": {"decision": {"bank_transfer": 0.966234564781189, "cash": 0.011082742363214493, "credit_card": 0.015811599791049957, "paypal": 0.00687115453183651}}, "native_timing": {"tokens": 53, "state_tokens": 11, "latency_ms": 71.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 54}, "server_wall_seconds": 0.07407845899999899, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "high", "confidence": 0.45, "probabilities": {"high": 0.59, "low": 0.02, "medium": 0.03, "urgent": 0.37}}}, "usage": {"input_tokens": 54, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"high": 0.5854946970939636, "low": 0.019567472860217094, "medium": 0.029012948274612427, "urgent": 0.3659248650074005}}, "native_timing": {"tokens": 54, "state_tokens": 19, "latency_ms": 73.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 52}, "server_wall_seconds": 0.07179941700000114, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "low", "confidence": 0.94, "probabilities": {"high": 0.01, "low": 0.95, "medium": 0.03, "urgent": 0.01}}}, "usage": {"input_tokens": 52, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"high": 0.005562879145145416, "low": 0.9528421759605408, "medium": 0.03297973796725273, "urgent": 0.008615217171609402}}, "native_timing": {"tokens": 52, "state_tokens": 17, "latency_ms": 70.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 52}, "server_wall_seconds": 0.07512938600000041, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "urgent", "confidence": 0.93, "probabilities": {"high": 0.03, "low": 0.01, "medium": 0.01, "urgent": 0.95}}}, "usage": {"input_tokens": 52, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"high": 0.031708426773548126, "low": 0.009729194454848766, "medium": 0.012577124871313572, "urgent": 0.9459853172302246}}, "native_timing": {"tokens": 52, "state_tokens": 17, "latency_ms": 74.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 45}, "server_wall_seconds": 0.07676813099999436, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "M", "confidence": 0.86, "probabilities": {"L": 0.02, "M": 0.9, "S": 0.06, "XL": 0.02}}}, "usage": {"input_tokens": 45, "output_tokens": 63}}, "unrounded_probabilities": {"decision": {"L": 0.023736421018838882, "M": 0.895587682723999, "S": 0.06452646106481552, "XL": 0.016149429604411125}}, "native_timing": {"tokens": 45, "state_tokens": 13, "latency_ms": 75.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 43}, "server_wall_seconds": 0.07002794800000345, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "XL", "confidence": 0.95, "probabilities": {"L": 0.02, "M": 0.01, "S": 0.01, "XL": 0.96}}}, "usage": {"input_tokens": 43, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"L": 0.019216492772102356, "M": 0.010408583097159863, "S": 0.006238934583961964, "XL": 0.964136004447937}}, "native_timing": {"tokens": 43, "state_tokens": 11, "latency_ms": 69.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 53}, "server_wall_seconds": 0.06621184600000163, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "thursday", "confidence": 0.9, "probabilities": {"friday": 0.04, "monday": 0.01, "thursday": 0.92, "tuesday": 0.02, "wednesday": 0.02}}}, "usage": {"input_tokens": 53, "output_tokens": 78}}, "unrounded_probabilities": {"decision": {"friday": 0.03580309450626373, "monday": 0.010451707988977432, "thursday": 0.9189052581787109, "tuesday": 0.017773400992155075, "wednesday": 0.01706649549305439}}, "native_timing": {"tokens": 53, "state_tokens": 11, "latency_ms": 65.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 55}, "server_wall_seconds": 0.06864762099999666, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "monday", "confidence": 0.76, "probabilities": {"friday": 0.03, "monday": 0.81, "thursday": 0.05, "tuesday": 0.05, "wednesday": 0.06}}}, "usage": {"input_tokens": 55, "output_tokens": 79}}, "unrounded_probabilities": {"decision": {"friday": 0.03251590207219124, "monday": 0.8080098628997803, "thursday": 0.0463910736143589, "tuesday": 0.049873556941747665, "wednesday": 0.06320958584547043}}, "native_timing": {"tokens": 55, "state_tokens": 13, "latency_ms": 67.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 51}, "server_wall_seconds": 0.06469985500000064, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "friday", "confidence": 0.88, "probabilities": {"friday": 0.91, "monday": 0.02, "thursday": 0.03, "tuesday": 0.02, "wednesday": 0.03}}}, "usage": {"input_tokens": 51, "output_tokens": 79}}, "unrounded_probabilities": {"decision": {"friday": 0.9064440727233887, "monday": 0.015017958357930183, "thursday": 0.03083491139113903, "tuesday": 0.019117753952741623, "wednesday": 0.028585240244865417}}, "native_timing": {"tokens": 51, "state_tokens": 9, "latency_ms": 64.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-extraction-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 56}, "server_wall_seconds": 0.07213242400000297, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "credit_card", "confidence": 0.95, "probabilities": {"bank_transfer": 0.01, "cash": 0.01, "credit_card": 0.96, "paypal": 0.01}}}, "usage": {"input_tokens": 56, "output_tokens": 67}}, "unrounded_probabilities": {"decision": {"bank_transfer": 0.012043376453220844, "cash": 0.013001848012208939, "credit_card": 0.9613389372825623, "paypal": 0.013615811243653297}}, "native_timing": {"tokens": 56, "state_tokens": 14, "latency_ms": 71.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-00", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 84}, "server_wall_seconds": 0.07282196300000265, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "get_weather", "confidence": 0.99, "probabilities": {"convert_currency": 0.0, "create_calendar_event": 0.0, "get_weather": 0.99, "send_email": 0.0, "translate_text": 0.0}}}, "usage": {"input_tokens": 84, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"convert_currency": 0.0006506135687232018, "create_calendar_event": 0.0011936001246795058, "get_weather": 0.99303138256073, "send_email": 0.003437215695157647, "translate_text": 0.0016871480038389564}}, "native_timing": {"tokens": 84, "state_tokens": 11, "latency_ms": 71.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 85}, "server_wall_seconds": 0.08304997700000172, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "send_email", "confidence": 0.99, "probabilities": {"convert_currency": 0.0, "create_calendar_event": 0.0, "get_weather": 0.0, "send_email": 0.99, "translate_text": 0.0}}}, "usage": {"input_tokens": 85, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"convert_currency": 0.0011770165292546153, "create_calendar_event": 0.0026351965498179197, "get_weather": 0.0021128994412720203, "send_email": 0.9898489713668823, "translate_text": 0.004225917626172304}}, "native_timing": {"tokens": 85, "state_tokens": 12, "latency_ms": 81.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 88}, "server_wall_seconds": 0.08159013800000281, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "create_calendar_event", "confidence": 0.99, "probabilities": {"convert_currency": 0.0, "create_calendar_event": 0.99, "get_weather": 0.0, "send_email": 0.0, "translate_text": 0.0}}}, "usage": {"input_tokens": 88, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"convert_currency": 0.00046877042041160166, "create_calendar_event": 0.9927728772163391, "get_weather": 0.0009144147043116391, "send_email": 0.0049062324687838554, "translate_text": 0.0009376758825965226}}, "native_timing": {"tokens": 88, "state_tokens": 15, "latency_ms": 80.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 86}, "server_wall_seconds": 0.07930267700000115, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "convert_currency", "confidence": 0.99, "probabilities": {"convert_currency": 0.99, "create_calendar_event": 0.0, "get_weather": 0.0, "send_email": 0.0, "translate_text": 0.01}}}, "usage": {"input_tokens": 86, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"convert_currency": 0.9880379438400269, "create_calendar_event": 0.0007025570957921445, "get_weather": 0.000935077725443989, "send_email": 0.0030461540445685387, "translate_text": 0.007278310135006905}}, "native_timing": {"tokens": 86, "state_tokens": 13, "latency_ms": 78.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 82}, "server_wall_seconds": 0.07999718599999994, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "translate_text", "confidence": 0.99, "probabilities": {"convert_currency": 0.0, "create_calendar_event": 0.0, "get_weather": 0.0, "send_email": 0.01, "translate_text": 0.99}}}, "usage": {"input_tokens": 82, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"convert_currency": 0.0014419001527130604, "create_calendar_event": 0.0014214224647730589, "get_weather": 0.0015844543231651187, "send_email": 0.005115137435495853, "translate_text": 0.9904370903968811}}, "native_timing": {"tokens": 82, "state_tokens": 9, "latency_ms": 78.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 88}, "server_wall_seconds": 0.0783252899999951, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "create_calendar_event", "confidence": 0.99, "probabilities": {"convert_currency": 0.0, "create_calendar_event": 0.99, "get_weather": 0.0, "send_email": 0.01, "translate_text": 0.0}}}, "usage": {"input_tokens": 88, "output_tokens": 78}}, "unrounded_probabilities": {"decision": {"convert_currency": 0.00024974337429739535, "create_calendar_event": 0.9923728108406067, "get_weather": 0.0007200388936325908, "send_email": 0.006151293404400349, "translate_text": 0.0005061687552370131}}, "native_timing": {"tokens": 88, "state_tokens": 15, "latency_ms": 77.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 92}, "server_wall_seconds": 0.08119797799999873, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "search_flights", "confidence": 0.99, "probabilities": {"book_restaurant": 0.0, "call_taxi": 0.0, "get_stock_price": 0.0, "search_flights": 0.99, "set_reminder": 0.01}}}, "usage": {"input_tokens": 92, "output_tokens": 82}}, "unrounded_probabilities": {"decision": {"book_restaurant": 0.0018395129591226578, "call_taxi": 0.004136552568525076, "get_stock_price": 0.0009688755962997675, "search_flights": 0.9880024194717407, "set_reminder": 0.005052694585174322}}, "native_timing": {"tokens": 92, "state_tokens": 14, "latency_ms": 80.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 92}, "server_wall_seconds": 0.08119759699999918, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "book_restaurant", "confidence": 0.95, "probabilities": {"book_restaurant": 0.96, "call_taxi": 0.01, "get_stock_price": 0.0, "search_flights": 0.0, "set_reminder": 0.03}}}, "usage": {"input_tokens": 92, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"book_restaurant": 0.956574022769928, "call_taxi": 0.007457044906914234, "get_stock_price": 0.0012105456553399563, "search_flights": 0.003913055639714003, "set_reminder": 0.030845284461975098}}, "native_timing": {"tokens": 92, "state_tokens": 14, "latency_ms": 80.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 91}, "server_wall_seconds": 0.07562261000000348, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "set_reminder", "confidence": 0.99, "probabilities": {"book_restaurant": 0.0, "call_taxi": 0.01, "get_stock_price": 0.0, "search_flights": 0.0, "set_reminder": 0.99}}}, "usage": {"input_tokens": 91, "output_tokens": 82}}, "unrounded_probabilities": {"decision": {"book_restaurant": 0.0013180827954784036, "call_taxi": 0.007674291264265776, "get_stock_price": 0.0011746537638828158, "search_flights": 0.0012332310434430838, "set_reminder": 0.9885998964309692}}, "native_timing": {"tokens": 91, "state_tokens": 13, "latency_ms": 74.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 88}, "server_wall_seconds": 0.07676591100000252, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "get_stock_price", "confidence": 1.0, "probabilities": {"book_restaurant": 0.0, "call_taxi": 0.0, "get_stock_price": 1.0, "search_flights": 0.0, "set_reminder": 0.0}}}, "usage": {"input_tokens": 88, "output_tokens": 79}}, "unrounded_probabilities": {"decision": {"book_restaurant": 0.0008003750117495656, "call_taxi": 0.0010489763226360083, "get_stock_price": 0.9961070418357849, "search_flights": 0.0005824934341944754, "set_reminder": 0.0014611025108024478}}, "native_timing": {"tokens": 88, "state_tokens": 10, "latency_ms": 75.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 88}, "server_wall_seconds": 0.07606695200000502, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "call_taxi", "confidence": 0.99, "probabilities": {"book_restaurant": 0.0, "call_taxi": 0.99, "get_stock_price": 0.0, "search_flights": 0.0, "set_reminder": 0.0}}}, "usage": {"input_tokens": 88, "output_tokens": 81}}, "unrounded_probabilities": {"decision": {"book_restaurant": 0.0007906971150077879, "call_taxi": 0.9949654936790466, "get_stock_price": 0.0005231574177742004, "search_flights": 0.0028736419044435024, "set_reminder": 0.0008470104658044875}}, "native_timing": {"tokens": 88, "state_tokens": 10, "latency_ms": 75.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:easy-tool_selection-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 91}, "server_wall_seconds": 0.07262513000000581, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "set_reminder", "confidence": 0.99, "probabilities": {"book_restaurant": 0.0, "call_taxi": 0.0, "get_stock_price": 0.0, "search_flights": 0.0, "set_reminder": 0.99}}}, "usage": {"input_tokens": 91, "output_tokens": 81}}, "unrounded_probabilities": {"decision": {"book_restaurant": 0.0014909126330167055, "call_taxi": 0.004482998047024012, "get_stock_price": 0.001196530763991177, "search_flights": 0.0010867260862141848, "set_reminder": 0.9917427897453308}}, "native_timing": {"tokens": 91, "state_tokens": 13, "latency_ms": 71.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 1618655232}
{"id": "benchmark:hard-opus-a-long_policy-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3897}, "server_wall_seconds": 0.3267680519999985, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "deny_vacancy_exclusion", "confidence": 0.07, "probabilities": {"deny_repeated_seepage": 0.14, "deny_vacancy_exclusion": 0.26, "pay_full_estimate_less_deductible": 0.23, "pay_subject_to_10000_sublimit": 0.18, "pay_subject_to_15000_sublimit": 0.19}}}, "usage": {"input_tokens": 3897, "output_tokens": 113}}, "unrounded_probabilities": {"decision": {"deny_repeated_seepage": 0.13780738413333893, "deny_vacancy_exclusion": 0.2565143406391144, "pay_full_estimate_less_deductible": 0.23389583826065063, "pay_subject_to_10000_sublimit": 0.182163268327713, "pay_subject_to_15000_sublimit": 0.18961913883686066}}, "native_timing": {"tokens": 3897, "state_tokens": 3666, "latency_ms": 320.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2590}, "server_wall_seconds": 0.22764849100000362, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "vp_and_finance_director", "confidence": 0.18, "probabilities": {"budget_holder": 0.14, "cfo": 0.14, "department_head": 0.27, "executive_committee": 0.11, "vp_and_finance_director": 0.35}}}, "usage": {"input_tokens": 2590, "output_tokens": 89}}, "unrounded_probabilities": {"decision": {"budget_holder": 0.1356707364320755, "cfo": 0.13816748559474945, "department_head": 0.26679977774620056, "executive_committee": 0.11241935938596725, "vp_and_finance_director": 0.34694257378578186}}, "native_timing": {"tokens": 2590, "state_tokens": 2396, "latency_ms": 219.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2364}, "server_wall_seconds": 0.20185328500000566, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "not_covered", "confidence": 0.21, "probabilities": {"full_repair_no_charge": 0.18, "not_covered": 0.41, "parts_covered_labour_charged": 0.27, "pro_rata_replacement_credit": 0.14}}}, "usage": {"input_tokens": 2364, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"full_repair_no_charge": 0.17757661640644073, "not_covered": 0.4099539816379547, "parts_covered_labour_charged": 0.27455922961235046, "pro_rata_replacement_credit": 0.1379101574420929}}, "native_timing": {"tokens": 2364, "state_tokens": 2226, "latency_ms": 197.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2492}, "server_wall_seconds": 0.20243599799999856, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "no_license_required", "confidence": 0.15, "probabilities": {"license_exception_lvs": 0.32, "license_required": 0.12, "no_license_required": 0.36, "shipment_prohibited": 0.2}}}, "usage": {"input_tokens": 2492, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"license_exception_lvs": 0.3175589442253113, "license_required": 0.12486491352319717, "no_license_required": 0.3617250621318817, "shipment_prohibited": 0.19585104286670685}}, "native_timing": {"tokens": 2492, "state_tokens": 2365, "latency_ms": 197.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2306}, "server_wall_seconds": 0.2105203950000032, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.29}}, "usage": {"input_tokens": 2306, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.7077447175979614, "true": 0.29225531220436096}}, "native_timing": {"tokens": 2306, "state_tokens": 2203, "latency_ms": 206.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-13", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2398}, "server_wall_seconds": 0.225201368999997, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.08}}, "usage": {"input_tokens": 2398, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.9201903343200684, "true": 0.07980959117412567}}, "native_timing": {"tokens": 2398, "state_tokens": 2309, "latency_ms": 220.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-17", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2324}, "server_wall_seconds": 0.2017709810000028, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "vendor_support", "confidence": 0.16, "probabilities": {"database_oncall": 0.14, "no_page_ticket_only": 0.27, "platform_oncall": 0.12, "security_incident_response": 0.14, "vendor_support": 0.33}}}, "usage": {"input_tokens": 2324, "output_tokens": 85}}, "unrounded_probabilities": {"decision": {"database_oncall": 0.1449994295835495, "no_page_ticket_only": 0.2676147520542145, "platform_oncall": 0.11769771575927734, "security_incident_response": 0.1392894834280014, "vendor_support": 0.3303985595703125}}, "native_timing": {"tokens": 2324, "state_tokens": 2209, "latency_ms": 197.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-long_policy-19", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2297}, "server_wall_seconds": 0.1935290320000007, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.79}}, "usage": {"input_tokens": 2297, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.2054438292980194, "true": 0.794556200504303}}, "native_timing": {"tokens": 2297, "state_tokens": 2217, "latency_ms": 189.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-probability-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 401}, "server_wall_seconds": 0.09075913499999899, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.71}}, "usage": {"input_tokens": 401, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.29460883140563965, "true": 0.7053911685943604}}, "native_timing": {"tokens": 401, "state_tokens": 337, "latency_ms": 89.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-probability-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 1070}, "server_wall_seconds": 0.12269983200000212, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "late", "confidence": 0.32, "probabilities": {"early": 0.22, "late": 0.55, "on_time": 0.23}}}, "usage": {"input_tokens": 1070, "output_tokens": 56}}, "unrounded_probabilities": {"decision": {"early": 0.21797029674053192, "late": 0.549335241317749, "on_time": 0.23269446194171906}}, "native_timing": {"tokens": 1070, "state_tokens": 1004, "latency_ms": 120.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-probability-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 1186}, "server_wall_seconds": 0.14760293999999874, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "reopened", "confidence": 0.09, "probabilities": {"escalated": 0.38, "reopened": 0.39, "resolved_first_contact": 0.22}}}, "usage": {"input_tokens": 1186, "output_tokens": 60}}, "unrounded_probabilities": {"decision": {"escalated": 0.3843245208263397, "reopened": 0.39280667901039124, "resolved_first_contact": 0.22286880016326904}}, "native_timing": {"tokens": 1186, "state_tokens": 1123, "latency_ms": 144.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-probability-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 363}, "server_wall_seconds": 0.0996530139999976, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.58}}, "usage": {"input_tokens": 363, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.4198921024799347, "true": 0.5801079273223877}}, "native_timing": {"tokens": 363, "state_tokens": 313, "latency_ms": 98.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-temporal_numeric-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 459}, "server_wall_seconds": 0.10616507900000016, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.14}}, "usage": {"input_tokens": 459, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.8608112335205078, "true": 0.1391887664794922}}, "native_timing": {"tokens": 459, "state_tokens": 393, "latency_ms": 104.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-temporal_numeric-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 494}, "server_wall_seconds": 0.10546491899999921, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.93}}, "usage": {"input_tokens": 494, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.071785569190979, "true": 0.928214430809021}}, "native_timing": {"tokens": 494, "state_tokens": 414, "latency_ms": 103.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-temporal_numeric-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 1269}, "server_wall_seconds": 0.13692529299999734, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "sep_24", "confidence": 0.12, "probabilities": {"sep_24": 0.29, "sep_25": 0.17, "sep_26": 0.14, "sep_27": 0.22, "sep_28": 0.17}}}, "usage": {"input_tokens": 1269, "output_tokens": 91}}, "unrounded_probabilities": {"decision": {"sep_24": 0.2932996451854706, "sep_25": 0.16746516525745392, "sep_26": 0.14449448883533478, "sep_27": 0.22180534899234772, "sep_28": 0.172935351729393}}, "native_timing": {"tokens": 1269, "state_tokens": 1177, "latency_ms": 134.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-temporal_numeric-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 491}, "server_wall_seconds": 0.09446460399999523, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.73}}, "usage": {"input_tokens": 491, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.2681545317173004, "true": 0.7318454384803772}}, "native_timing": {"tokens": 491, "state_tokens": 409, "latency_ms": 93.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-a-temporal_numeric-12", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 550}, "server_wall_seconds": 0.10027705999999625, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 2.12, "legend": {"0": "No violations", "1": "Exactly one violation", "2": "Exactly two violations", "3": "Three or more violations"}, "probabilities": {"0": 0.11, "1": 0.12, "2": 0.31, "3": 0.46}, "confidence": 0.71}}, "usage": {"input_tokens": 550, "output_tokens": 102}}, "unrounded_probabilities": {"decision": {"0": 0.10704302787780762, "1": 0.12230099737644196, "2": 0.3145110607147217, "3": 0.45614489912986755}}, "native_timing": {"tokens": 550, "state_tokens": 493, "latency_ms": 98.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 506}, "server_wall_seconds": 0.09644721699999792, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "false_positive", "confidence": 0.06, "probabilities": {"cannot_determine": 0.28, "false_positive": 0.37, "real_incident": 0.34}}}, "usage": {"input_tokens": 506, "output_tokens": 61}}, "unrounded_probabilities": {"decision": {"cannot_determine": 0.28440985083580017, "false_positive": 0.3729357421398163, "real_incident": 0.34265437722206116}}, "native_timing": {"tokens": 506, "state_tokens": 428, "latency_ms": 95.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 461}, "server_wall_seconds": 0.10011581499999522, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "insufficient_information", "confidence": 0.42, "probabilities": {"insufficient_information": 0.62, "time_barred": 0.32, "within_limitation": 0.06}}}, "usage": {"input_tokens": 461, "output_tokens": 63}}, "unrounded_probabilities": {"decision": {"insufficient_information": 0.6162480115890503, "time_barred": 0.320000559091568, "within_limitation": 0.06375143676996231}}, "native_timing": {"tokens": 461, "state_tokens": 367, "latency_ms": 98.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 464}, "server_wall_seconds": 0.11642810299999695, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "over_limit", "confidence": 0.28, "probabilities": {"insufficient_information": 0.45, "over_limit": 0.52, "within_policy": 0.03}}}, "usage": {"input_tokens": 464, "output_tokens": 60}}, "unrounded_probabilities": {"decision": {"insufficient_information": 0.45400914549827576, "over_limit": 0.5192869305610657, "within_policy": 0.026703884825110435}}, "native_timing": {"tokens": 464, "state_tokens": 389, "latency_ms": 114.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 604}, "server_wall_seconds": 0.1113550369999956, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "compromised", "confidence": 0.83, "probabilities": {"compromised": 0.89, "inconclusive": 0.09, "legitimate": 0.02}}}, "usage": {"input_tokens": 604, "output_tokens": 61}}, "unrounded_probabilities": {"decision": {"compromised": 0.8866763114929199, "inconclusive": 0.09170122444629669, "legitimate": 0.021622391417622566}}, "native_timing": {"tokens": 604, "state_tokens": 527, "latency_ms": 109.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 403}, "server_wall_seconds": 0.10332827199999883, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "send_to_alex_moreno", "confidence": 0.78, "probabilities": {"ask_user_which_alex": 0.07, "send_to_alex_moreno": 0.86, "send_to_alexandra_lind": 0.08}}}, "usage": {"input_tokens": 403, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"ask_user_which_alex": 0.06587604433298111, "send_to_alex_moreno": 0.8560175895690918, "send_to_alexandra_lind": 0.07810629159212112}}, "native_timing": {"tokens": 403, "state_tokens": 333, "latency_ms": 100.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 430}, "server_wall_seconds": 0.10055716799999459, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "grade_correct", "confidence": 0.27, "probabilities": {"cannot_determine": 0.31, "grade_correct": 0.51, "grade_should_be_higher": 0.18}}}, "usage": {"input_tokens": 430, "output_tokens": 63}}, "unrounded_probabilities": {"decision": {"cannot_determine": 0.30872076749801636, "grade_correct": 0.5117319226264954, "grade_should_be_higher": 0.1795472800731659}}, "native_timing": {"tokens": 430, "state_tokens": 354, "latency_ms": 98.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-ambiguous-13", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 567}, "server_wall_seconds": 0.10399467900000303, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "cannot_determine", "confidence": 0.31, "probabilities": {"cannot_determine": 0.48, "charge_data_team": 0.2, "charge_platform_team": 0.2, "split_50_50": 0.11}}}, "usage": {"input_tokens": 567, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"cannot_determine": 0.48377543687820435, "charge_data_team": 0.20090962946414948, "charge_platform_team": 0.20210576057434082, "split_50_50": 0.11320915818214417}}, "native_timing": {"tokens": 567, "state_tokens": 452, "latency_ms": 102.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-multi_hop-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2930}, "server_wall_seconds": 0.2312185850000006, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "tier2_department_head", "confidence": 0.41, "probabilities": {"tier1_team_lead": 0.12, "tier2_department_head": 0.56, "tier3_cfo": 0.22, "tier4_board": 0.1}}}, "usage": {"input_tokens": 2930, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"tier1_team_lead": 0.11677651852369308, "tier2_department_head": 0.5606659054756165, "tier3_cfo": 0.21938151121139526, "tier4_board": 0.10317610949277878}}, "native_timing": {"tokens": 2930, "state_tokens": 2840, "latency_ms": 222.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-multi_hop-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2312}, "server_wall_seconds": 0.21396804300000127, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "dmitri", "confidence": 0.26, "probabilities": {"ana": 0.11, "bjorn": 0.12, "chen": 0.11, "dmitri": 0.38, "esra": 0.09, "no_page_ticket_only": 0.18}}}, "usage": {"input_tokens": 2312, "output_tokens": 91}}, "unrounded_probabilities": {"decision": {"ana": 0.11003507673740387, "bjorn": 0.11926865577697754, "chen": 0.11183587461709976, "dmitri": 0.38250038027763367, "esra": 0.09399048238992691, "no_page_ticket_only": 0.18236948549747467}}, "native_timing": {"tokens": 2312, "state_tokens": 2227, "latency_ms": 209.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-multi_hop-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2542}, "server_wall_seconds": 0.2314951329999957, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.95, "legend": {"0": "Level 0 — no award", "1": "Level 1 — EUR 500", "2": "Level 2 — EUR 1,200", "3": "Level 3 — EUR 2,000", "4": "Level 4 — EUR 3,000"}, "probabilities": {"0": 0.19, "1": 0.16, "2": 0.31, "3": 0.18, "4": 0.15}, "confidence": 0.74}}, "usage": {"input_tokens": 2542, "output_tokens": 177}}, "unrounded_probabilities": {"decision": {"0": 0.1934874951839447, "1": 0.1559142917394638, "2": 0.31186726689338684, "3": 0.18477857112884521, "4": 0.15395234525203705}}, "native_timing": {"tokens": 2542, "state_tokens": 2452, "latency_ms": 225.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-multi_hop-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2607}, "server_wall_seconds": 0.22841121000000442, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "credit_10_percent", "confidence": 0.36, "probabilities": {"credit_10_percent": 0.47, "credit_20_percent": 0.19, "credit_25_percent": 0.09, "credit_50_percent": 0.09, "credit_full_month": 0.07, "no_credit": 0.09}}}, "usage": {"input_tokens": 2607, "output_tokens": 105}}, "unrounded_probabilities": {"decision": {"credit_10_percent": 0.47048524022102356, "credit_20_percent": 0.19281575083732605, "credit_25_percent": 0.08652066439390182, "credit_50_percent": 0.0856575295329094, "credit_full_month": 0.07254254072904587, "no_credit": 0.09197824448347092}}, "native_timing": {"tokens": 2607, "state_tokens": 2460, "latency_ms": 223.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-multi_hop-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2284}, "server_wall_seconds": 0.19884828, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.59}}, "usage": {"input_tokens": 2284, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.41199755668640137, "true": 0.5880025029182434}}, "native_timing": {"tokens": 2284, "state_tokens": 2190, "latency_ms": 194.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-probability-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 1885}, "server_wall_seconds": 0.17746347799999995, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "merchant_wins", "confidence": 0.21, "probabilities": {"customer_wins": 0.41, "customer_withdraws": 0.12, "merchant_wins": 0.48}}}, "usage": {"input_tokens": 1885, "output_tokens": 64}}, "unrounded_probabilities": {"decision": {"customer_wins": 0.40680351853370667, "customer_withdraws": 0.11753184348344803, "merchant_wins": 0.4756646752357483}}, "native_timing": {"tokens": 1885, "state_tokens": 1804, "latency_ms": 174.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-probability-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 467}, "server_wall_seconds": 0.10571293399999604, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "bad_push", "confidence": 0.51, "probabilities": {"bad_push": 0.67, "database_hardware": 0.13, "upstream_provider": 0.2}}}, "usage": {"input_tokens": 467, "output_tokens": 60}}, "unrounded_probabilities": {"decision": {"bad_push": 0.6708141565322876, "database_hardware": 0.13199232518672943, "upstream_provider": 0.19719350337982178}}, "native_timing": {"tokens": 467, "state_tokens": 397, "latency_ms": 104.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-probability-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 1185}, "server_wall_seconds": 0.1546887570000024, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.37}}, "usage": {"input_tokens": 1185, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.6256071925163269, "true": 0.3743928074836731}}, "native_timing": {"tokens": 1185, "state_tokens": 1125, "latency_ms": 151.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-probability-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 568}, "server_wall_seconds": 0.11137235599999684, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "no_show", "confidence": 0.08, "probabilities": {"attended": 0.38, "late_cancellation": 0.23, "no_show": 0.39}}}, "usage": {"input_tokens": 568, "output_tokens": 59}}, "unrounded_probabilities": {"decision": {"attended": 0.3794598877429962, "late_cancellation": 0.23318985104560852, "no_show": 0.3873502314090729}}, "native_timing": {"tokens": 568, "state_tokens": 473, "latency_ms": 109.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-probability-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 987}, "server_wall_seconds": 0.13611011900000136, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "overturned", "confidence": 0.87, "probabilities": {"modified": 0.04, "overturned": 0.91, "upheld": 0.05}}}, "usage": {"input_tokens": 987, "output_tokens": 58}}, "unrounded_probabilities": {"decision": {"modified": 0.03856784850358963, "overturned": 0.9132384061813354, "upheld": 0.0481937900185585}}, "native_timing": {"tokens": 987, "state_tokens": 921, "latency_ms": 133.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-tradeoff-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 808}, "server_wall_seconds": 0.12166271400000284, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "escalate_to_engineering", "confidence": 0.66, "probabilities": {"escalate_to_engineering": 0.73, "escalate_to_security": 0.12, "page_oncall": 0.06, "process_refund": 0.05, "send_kb_article": 0.05}}}, "usage": {"input_tokens": 808, "output_tokens": 90}}, "unrounded_probabilities": {"decision": {"escalate_to_engineering": 0.7289229035377502, "escalate_to_security": 0.11565219610929489, "page_oncall": 0.05879027768969536, "process_refund": 0.05019955337047577, "send_kb_article": 0.04643508791923523}}, "native_timing": {"tokens": 808, "state_tokens": 676, "latency_ms": 119.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-tradeoff-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 725}, "server_wall_seconds": 0.11248486999999585, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "order_c", "confidence": 0.26, "probabilities": {"order_a": 0.12, "order_b": 0.21, "order_c": 0.44, "order_d": 0.23}}}, "usage": {"input_tokens": 725, "output_tokens": 69}}, "unrounded_probabilities": {"decision": {"order_a": 0.11761078238487244, "order_b": 0.21316738426685333, "order_c": 0.4426765739917755, "order_d": 0.2265452742576599}}, "native_timing": {"tokens": 725, "state_tokens": 628, "latency_ms": 110.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-tradeoff-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 762}, "server_wall_seconds": 0.11191193500000196, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.26}}, "usage": {"input_tokens": 762, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.7368577718734741, "true": 0.2631422281265259}}, "native_timing": {"tokens": 762, "state_tokens": 678, "latency_ms": 110.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-tradeoff-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 684}, "server_wall_seconds": 0.11047221599999801, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "p2_fix_30d", "confidence": 0.56, "probabilities": {"out_of_scope": 0.04, "p0_fix_24h": 0.11, "p1_fix_7d": 0.1, "p2_fix_30d": 0.65, "p3_backlog": 0.1}}}, "usage": {"input_tokens": 684, "output_tokens": 99}}, "unrounded_probabilities": {"decision": {"out_of_scope": 0.04095775634050369, "p0_fix_24h": 0.10578594356775284, "p1_fix_7d": 0.09867757558822632, "p2_fix_30d": 0.6519199013710022, "p3_backlog": 0.10265889018774033}}, "native_timing": {"tokens": 684, "state_tokens": 549, "latency_ms": 108.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-tradeoff-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 573}, "server_wall_seconds": 0.11827568600000404, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.24}}, "usage": {"input_tokens": 573, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.7584527730941772, "true": 0.24154725670814514}}, "native_timing": {"tokens": 573, "state_tokens": 488, "latency_ms": 116.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-b-tradeoff-12", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 560}, "server_wall_seconds": 0.10930390500000442, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "wait_for_inspection", "confidence": 0.18, "probabilities": {"deny_refund": 0.28, "escalate_to_fraud_team": 0.04, "offer_store_credit": 0.09, "refund_now": 0.25, "wait_for_inspection": 0.34}}}, "usage": {"input_tokens": 560, "output_tokens": 89}}, "unrounded_probabilities": {"decision": {"deny_refund": 0.2791862487792969, "escalate_to_fraud_team": 0.04250903055071831, "offer_store_credit": 0.08905017375946045, "refund_now": 0.24871502816677094, "wait_for_inspection": 0.3405395746231079}}, "native_timing": {"tokens": 560, "state_tokens": 458, "latency_ms": 107.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3549}, "server_wall_seconds": 0.28279024200000435, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "restrict_all_pending_legal_hold", "confidence": 0.1, "probabilities": {"erase_all": 0.26, "erase_but_retain_transaction_records": 0.17, "erase_marketing_only": 0.13, "reject_unverified_request": 0.16, "restrict_all_pending_legal_hold": 0.28}}}, "usage": {"input_tokens": 3549, "output_tokens": 92}}, "unrounded_probabilities": {"decision": {"erase_all": 0.2613794207572937, "erase_but_retain_transaction_records": 0.17493504285812378, "erase_marketing_only": 0.12633474171161652, "reject_unverified_request": 0.1557231992483139, "restrict_all_pending_legal_hold": 0.2816275656223297}}, "native_timing": {"tokens": 3549, "state_tokens": 3377, "latency_ms": 271.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3473}, "server_wall_seconds": 0.2563591850000009, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "eur_0", "confidence": 0.1, "probabilities": {"eur_0": 0.25, "eur_150": 0.17, "eur_200": 0.17, "eur_300": 0.2, "eur_400": 0.09, "eur_600": 0.12}}}, "usage": {"input_tokens": 3473, "output_tokens": 104}}, "unrounded_probabilities": {"decision": {"eur_0": 0.2496323585510254, "eur_150": 0.17080830037593842, "eur_200": 0.1654399037361145, "eur_300": 0.20441919565200806, "eur_400": 0.08728175610303879, "eur_600": 0.12241854518651962}}, "native_timing": {"tokens": 3473, "state_tokens": 3343, "latency_ms": 250.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3284}, "server_wall_seconds": 0.2530256040000012, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.42}}, "usage": {"input_tokens": 3284, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.5794521570205688, "true": 0.42054784297943115}}, "native_timing": {"tokens": 3284, "state_tokens": 3194, "latency_ms": 244.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3202}, "server_wall_seconds": 0.2675718850000024, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.12, "legend": {"0": "Compliant: Named Users do not exceed Licences held on the Measurement Date", "1": "Shortfall of 1 to 5 Licences", "2": "Shortfall of 6 to 15 Licences", "3": "Shortfall of more than 15 Licences"}, "probabilities": {"0": 0.38, "1": 0.25, "2": 0.24, "3": 0.13}, "confidence": 0.63}}, "usage": {"input_tokens": 3202, "output_tokens": 136}}, "unrounded_probabilities": {"decision": {"0": 0.38033759593963623, "1": 0.251056969165802, "2": 0.24091221392154694, "3": 0.12769322097301483}}, "native_timing": {"tokens": 3202, "state_tokens": 3112, "latency_ms": 261.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3206}, "server_wall_seconds": 0.25726012799999864, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.55, "legend": {"0": "SEV-4 (lowest)", "1": "SEV-3", "2": "SEV-2", "3": "SEV-1 (highest)"}, "probabilities": {"0": 0.14, "1": 0.29, "2": 0.44, "3": 0.13}, "confidence": 0.76}}, "usage": {"input_tokens": 3206, "output_tokens": 111}}, "unrounded_probabilities": {"decision": {"0": 0.1425655484199524, "1": 0.29393258690834045, "2": 0.4366069436073303, "3": 0.12689493596553802}}, "native_timing": {"tokens": 3206, "state_tokens": 3113, "latency_ms": 250.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3600}, "server_wall_seconds": 0.2639475110000049, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "reject_return", "confidence": 0.05, "probabilities": {"credit_minus_restocking_fee": 0.28, "full_credit": 0.2, "reject_return": 0.28, "repair_or_replace_only": 0.24}}}, "usage": {"input_tokens": 3600, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"credit_minus_restocking_fee": 0.2836270034313202, "full_credit": 0.1966577172279358, "reject_return": 0.28424426913261414, "repair_or_replace_only": 0.2354709953069687}}, "native_timing": {"tokens": 3600, "state_tokens": 3483, "latency_ms": 258.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-long_policy-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 3371}, "server_wall_seconds": 0.266060447000001, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.65}}, "usage": {"input_tokens": 3371, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.350205659866333, "true": 0.6497942805290222}}, "native_timing": {"tokens": 3371, "state_tokens": 3297, "latency_ms": 260.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-probability-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 516}, "server_wall_seconds": 0.11724805499999746, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "ortega_seal_works", "confidence": 0.18, "probabilities": {"halvorsen_polymer": 0.2, "kuznets_technik": 0.34, "ortega_seal_works": 0.46}}}, "usage": {"input_tokens": 516, "output_tokens": 72}}, "unrounded_probabilities": {"decision": {"halvorsen_polymer": 0.20398810505867004, "kuznets_technik": 0.34002307057380676, "ortega_seal_works": 0.4559888541698456}}, "native_timing": {"tokens": 516, "state_tokens": 426, "latency_ms": 115.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-temporal_numeric-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 908}, "server_wall_seconds": 0.13863723999999422, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.57}}, "usage": {"input_tokens": 908, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.4340916574001312, "true": 0.5659083724021912}}, "native_timing": {"tokens": 908, "state_tokens": 799, "latency_ms": 136.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-temporal_numeric-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 835}, "server_wall_seconds": 0.13665781600000315, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "eur_617_00", "confidence": 0.09, "probabilities": {"eur_617_00": 0.28, "eur_688_00": 0.19, "eur_692_00": 0.19, "eur_693_90": 0.21, "eur_812_00": 0.13}}}, "usage": {"input_tokens": 835, "output_tokens": 115}}, "unrounded_probabilities": {"decision": {"eur_617_00": 0.2758154571056366, "eur_688_00": 0.18626686930656433, "eur_692_00": 0.19176477193832397, "eur_693_90": 0.21155278384685516, "eur_812_00": 0.13460007309913635}}, "native_timing": {"tokens": 835, "state_tokens": 697, "latency_ms": 133.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-temporal_numeric-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 750}, "server_wall_seconds": 0.10976730499999832, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "usd_2222_50", "confidence": 0.04, "probabilities": {"usd_2222_50": 0.23, "usd_2303_01": 0.21, "usd_2303_33": 0.16, "usd_2335_00": 0.22, "usd_2340_00": 0.18}}}, "usage": {"input_tokens": 750, "output_tokens": 127}}, "unrounded_probabilities": {"decision": {"usd_2222_50": 0.22965408861637115, "usd_2303_01": 0.2094644457101822, "usd_2303_33": 0.15947362780570984, "usd_2335_00": 0.2243468165397644, "usd_2340_00": 0.1770610809326172}}, "native_timing": {"tokens": 750, "state_tokens": 571, "latency_ms": 107.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-temporal_numeric-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 952}, "server_wall_seconds": 0.12531062200000065, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "none_declined", "confidence": 0.1, "probabilities": {"none_declined": 0.25, "sep_15_office_chair": 0.16, "sep_16_taxi": 0.12, "sep_17_parking": 0.12, "sep_18_book_set": 0.14, "sep_21_client_dinner": 0.2}}}, "usage": {"input_tokens": 952, "output_tokens": 111}}, "unrounded_probabilities": {"decision": {"none_declined": 0.2533932328224182, "sep_15_office_chair": 0.16338367760181427, "sep_16_taxi": 0.12173942476511002, "sep_17_parking": 0.12418917566537857, "sep_18_book_set": 0.13621139526367188, "sep_21_client_dinner": 0.20108312368392944}}, "native_timing": {"tokens": 952, "state_tokens": 766, "latency_ms": 122.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-temporal_numeric-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 954}, "server_wall_seconds": 0.12432315600000265, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "eur_497_40", "confidence": 0.14, "probabilities": {"eur_497_40": 0.31, "eur_498_81": 0.28, "eur_506_06": 0.14, "eur_513_69": 0.16, "eur_532_01": 0.11}}}, "usage": {"input_tokens": 954, "output_tokens": 115}}, "unrounded_probabilities": {"decision": {"eur_497_40": 0.31191301345825195, "eur_498_81": 0.2779386341571808, "eur_506_06": 0.13975103199481964, "eur_513_69": 0.1603330671787262, "eur_532_01": 0.11006425321102142}}, "native_timing": {"tokens": 954, "state_tokens": 805, "latency_ms": 121.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-opus-c-temporal_numeric-12", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 790}, "server_wall_seconds": 0.12310188300000391, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "expired_24_month_term", "confidence": 0.13, "probabilities": {"covered": 0.29, "expired_24_month_term": 0.42, "expired_30_month_cap": 0.29}}}, "usage": {"input_tokens": 790, "output_tokens": 70}}, "unrounded_probabilities": {"decision": {"covered": 0.2880264222621918, "expired_24_month_term": 0.41821902990341187, "expired_30_month_cap": 0.29375457763671875}}, "native_timing": {"tokens": 790, "state_tokens": 685, "latency_ms": 120.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-adversarial-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 248}, "server_wall_seconds": 0.09220569400000045, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "deny_missing_approval", "confidence": 0.17, "probabilities": {"deny_inactive_user": 0.15, "deny_missing_approval": 0.38, "grant_access": 0.1, "grant_owner_exception": 0.37}}}, "usage": {"input_tokens": 248, "output_tokens": 72}}, "unrounded_probabilities": {"decision": {"deny_inactive_user": 0.1526007056236267, "deny_missing_approval": 0.3798134922981262, "grant_access": 0.09867750108242035, "grant_owner_exception": 0.3689083158969879}}, "native_timing": {"tokens": 248, "state_tokens": 189, "latency_ms": 90.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-adversarial-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 212}, "server_wall_seconds": 0.09272684800000519, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.19}}, "usage": {"input_tokens": 212, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.810388445854187, "true": 0.18961156904697418}}, "native_timing": {"tokens": 212, "state_tokens": 167, "latency_ms": 91.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-adversarial-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 264}, "server_wall_seconds": 0.0975137470000007, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "delete_now", "confidence": 0.54, "probabilities": {"delete_now": 0.65, "needs_signature_review": 0.1, "retain_indefinitely": 0.16, "retain_until_2031": 0.09}}}, "usage": {"input_tokens": 264, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"delete_now": 0.6533204913139343, "needs_signature_review": 0.09972269833087921, "retain_indefinitely": 0.15639328956604004, "retain_until_2031": 0.09056355804204941}}, "native_timing": {"tokens": 264, "state_tokens": 190, "latency_ms": 95.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-adversarial-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 189}, "server_wall_seconds": 0.0874848370000052, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.77}}, "usage": {"input_tokens": 189, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.23271076381206512, "true": 0.7672892808914185}}, "native_timing": {"tokens": 189, "state_tokens": 153, "latency_ms": 86.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-adversarial-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 189}, "server_wall_seconds": 0.08369879600000019, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 1.98, "legend": {"0": "Informational only.", "1": "Routine degradation.", "2": "Major multi-customer degradation.", "3": "Complete outage."}, "probabilities": {"0": 0.14, "1": 0.11, "2": 0.38, "3": 0.37}, "confidence": 0.74}}, "usage": {"input_tokens": 189, "output_tokens": 103}}, "unrounded_probabilities": {"decision": {"0": 0.14394040405750275, "1": 0.10754276812076569, "2": 0.3776400089263916, "3": 0.37087687849998474}}, "native_timing": {"tokens": 189, "state_tokens": 156, "latency_ms": 82.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-adversarial-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 270}, "server_wall_seconds": 0.08561069599999627, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "standard_research", "confidence": 0.57, "probabilities": {"ethics_review": 0.04, "major_grants": 0.11, "manual_hold": 0.18, "standard_research": 0.67}}}, "usage": {"input_tokens": 270, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"ethics_review": 0.03911445662379265, "major_grants": 0.11045869439840317, "manual_hold": 0.1758311539888382, "standard_research": 0.6745956540107727}}, "native_timing": {"tokens": 270, "state_tokens": 196, "latency_ms": 84.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2261}, "server_wall_seconds": 0.19442051499999735, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "reject_cutoff", "confidence": 0.08, "probabilities": {"accept_s2_dusk": 0.26, "reject_cutoff": 0.31, "reject_tier": 0.21, "reroute_s3": 0.22}}}, "usage": {"input_tokens": 2261, "output_tokens": 75}}, "unrounded_probabilities": {"decision": {"accept_s2_dusk": 0.25737255811691284, "reject_cutoff": 0.31208741664886475, "reject_tier": 0.2069786936044693, "reroute_s3": 0.2235613465309143}}, "native_timing": {"tokens": 2261, "state_tokens": 2190, "latency_ms": 190.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2265}, "server_wall_seconds": 0.21569345500000026, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "insufficient_information", "confidence": 0.45, "probabilities": {"executive_bridge": 0.07, "heron_q2": 0.22, "insufficient_information": 0.59, "security_primary": 0.12}}}, "usage": {"input_tokens": 2265, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"executive_bridge": 0.07075852155685425, "heron_q2": 0.22281071543693542, "insufficient_information": 0.590750515460968, "security_primary": 0.11568022519350052}}, "native_timing": {"tokens": 2265, "state_tokens": 2202, "latency_ms": 211.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2204}, "server_wall_seconds": 0.2078803749999949, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "reject_category", "confidence": 0.2, "probabilities": {"pay_manager_approved": 0.27, "reject_category": 0.4, "require_director": 0.21, "require_finance_board": 0.12}}}, "usage": {"input_tokens": 2204, "output_tokens": 71}}, "unrounded_probabilities": {"decision": {"pay_manager_approved": 0.27473148703575134, "reject_category": 0.39653950929641724, "require_director": 0.2096717208623886, "require_finance_board": 0.11905721575021744}}, "native_timing": {"tokens": 2204, "state_tokens": 2143, "latency_ms": 203.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2197}, "server_wall_seconds": 0.2102701689999975, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "manual_equivalency", "confidence": 0.12, "probabilities": {"admit": 0.09, "deny_prerequisite": 0.28, "deny_score": 0.29, "manual_equivalency": 0.34}}}, "usage": {"input_tokens": 2197, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"admit": 0.08932296931743622, "deny_prerequisite": 0.2836725413799286, "deny_score": 0.2868422269821167, "manual_equivalency": 0.3401622474193573}}, "native_timing": {"tokens": 2197, "state_tokens": 2133, "latency_ms": 205.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2173}, "server_wall_seconds": 0.1888839169999983, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "insufficient_log", "confidence": 0.12, "probabilities": {"central_engineering": 0.26, "insufficient_log": 0.34, "local_maintenance": 0.16, "safety_shutdown": 0.24}}}, "usage": {"input_tokens": 2173, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"central_engineering": 0.25760459899902344, "insufficient_log": 0.3368498384952545, "local_maintenance": 0.16148585081100464, "safety_shutdown": 0.2440596967935562}}, "native_timing": {"tokens": 2173, "state_tokens": 2101, "latency_ms": 183.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2172}, "server_wall_seconds": 0.19241244300000204, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "switch_l4", "confidence": 0.13, "probabilities": {"confirm_l3": 0.18, "reject_capacity": 0.25, "require_steward": 0.23, "switch_l4": 0.35}}}, "usage": {"input_tokens": 2172, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"confirm_l3": 0.17513254284858704, "reject_capacity": 0.24653971195220947, "require_steward": 0.23046796023845673, "switch_l4": 0.34785982966423035}}, "native_timing": {"tokens": 2172, "state_tokens": 2101, "latency_ms": 187.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2230}, "server_wall_seconds": 0.2048067859999989, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "deny_sensitive", "confidence": 0.17, "probabilities": {"approve_45_days": 0.18, "deny_sensitive": 0.38, "needs_field_review": 0.18, "reduce_to_30_days": 0.26}}}, "usage": {"input_tokens": 2230, "output_tokens": 77}}, "unrounded_probabilities": {"decision": {"approve_45_days": 0.18158148229122162, "deny_sensitive": 0.3798380494117737, "needs_field_review": 0.18278971314430237, "reduce_to_30_days": 0.25579071044921875}}, "native_timing": {"tokens": 2230, "state_tokens": 2161, "latency_ms": 199.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-multi_hop-12", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2179}, "server_wall_seconds": 0.21529882699999803, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "wrong_issue_class", "confidence": 0.17, "probabilities": {"covered_recurrence": 0.17, "expired": 0.24, "needs_authentication": 0.21, "wrong_issue_class": 0.38}}}, "usage": {"input_tokens": 2179, "output_tokens": 71}}, "unrounded_probabilities": {"decision": {"covered_recurrence": 0.1724662184715271, "expired": 0.23913638293743134, "needs_authentication": 0.20922034978866577, "wrong_issue_class": 0.379177063703537}}, "native_timing": {"tokens": 2179, "state_tokens": 2110, "latency_ms": 210.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 263}, "server_wall_seconds": 0.10586815999999999, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.26}}, "usage": {"input_tokens": 263, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.7434099316596985, "true": 0.2565900385379791}}, "native_timing": {"tokens": 263, "state_tokens": 220, "latency_ms": 104.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 169}, "server_wall_seconds": 0.09631472299999899, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.27}}, "usage": {"input_tokens": 169, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.730274498462677, "true": 0.2697254419326782}}, "native_timing": {"tokens": 169, "state_tokens": 130, "latency_ms": 94.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 299}, "server_wall_seconds": 0.09046990699999924, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "score", "score": 0.58, "legend": {"0": "No operative exception.", "1": "Requested but unauthorized exception.", "2": "Authorized, unused exception.", "3": "Authorized exception already used."}, "probabilities": {"0": 0.71, "1": 0.08, "2": 0.13, "3": 0.08}, "confidence": 0.81}}, "usage": {"input_tokens": 299, "output_tokens": 106}}, "unrounded_probabilities": {"decision": {"0": 0.7118715643882751, "1": 0.08202363550662994, "2": 0.12529277801513672, "3": 0.08081195503473282}}, "native_timing": {"tokens": 299, "state_tokens": 262, "latency_ms": 88.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 316}, "server_wall_seconds": 0.09010370699999726, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "close_confirmed", "confidence": 0.17, "probabilities": {"await_internal_metrics": 0.17, "close_agent_verified": 0.18, "close_confirmed": 0.38, "keep_open_unresolved": 0.28}}}, "usage": {"input_tokens": 316, "output_tokens": 75}}, "unrounded_probabilities": {"decision": {"await_internal_metrics": 0.16875363886356354, "close_agent_verified": 0.17773132026195526, "close_confirmed": 0.37548530101776123, "keep_open_unresolved": 0.27802973985671997}}, "native_timing": {"tokens": 316, "state_tokens": 244, "latency_ms": 88.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 244}, "server_wall_seconds": 0.08701652399999915, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "on_time", "confidence": 0.33, "probabilities": {"canceled": 0.05, "indeterminate": 0.22, "late": 0.23, "on_time": 0.5}}}, "usage": {"input_tokens": 244, "output_tokens": 67}}, "unrounded_probabilities": {"decision": {"canceled": 0.05341067537665367, "indeterminate": 0.22144946455955505, "late": 0.2257717251777649, "on_time": 0.49936816096305847}}, "native_timing": {"tokens": 244, "state_tokens": 174, "latency_ms": 85.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 170}, "server_wall_seconds": 0.08062349200000085, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "blocked_second_review", "confidence": 0.49, "probabilities": {"blocked_failed_test": 0.15, "blocked_second_review": 0.62, "ready_one_review": 0.15, "ready_two_reviews": 0.08}}}, "usage": {"input_tokens": 170, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"blocked_failed_test": 0.14685268700122833, "blocked_second_review": 0.6176565289497375, "ready_one_review": 0.15076839923858643, "ready_two_reviews": 0.08472247421741486}}, "native_timing": {"tokens": 170, "state_tokens": 103, "latency_ms": 79.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-13", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 177}, "server_wall_seconds": 0.09325679100000173, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.09}}, "usage": {"input_tokens": 177, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.909955620765686, "true": 0.09004437923431396}}, "native_timing": {"tokens": 177, "state_tokens": 132, "latency_ms": 91.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-a-trap-15", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 279}, "server_wall_seconds": 0.09906521799999979, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.31}}, "usage": {"input_tokens": 279, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.6947142481803894, "true": 0.3052857220172882}}, "native_timing": {"tokens": 279, "state_tokens": 232, "latency_ms": 97.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 255}, "server_wall_seconds": 0.08895953700000092, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.87}}, "usage": {"input_tokens": 255, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.1315310150384903, "true": 0.8684689998626709}}, "native_timing": {"tokens": 255, "state_tokens": 199, "latency_ms": 87.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 179}, "server_wall_seconds": 0.09009608700000626, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.86}}, "usage": {"input_tokens": 179, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.14110992848873138, "true": 0.8588901162147522}}, "native_timing": {"tokens": 179, "state_tokens": 123, "latency_ms": 88.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 194}, "server_wall_seconds": 0.09011387700000029, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.9}}, "usage": {"input_tokens": 194, "output_tokens": 19}}, "unrounded_probabilities": {"decision": {"false": 0.10145549476146698, "true": 0.8985445499420166}}, "native_timing": {"tokens": 194, "state_tokens": 138, "latency_ms": 88.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 181}, "server_wall_seconds": 0.09492119600000137, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.88}}, "usage": {"input_tokens": 181, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.11919458955526352, "true": 0.8808054327964783}}, "native_timing": {"tokens": 181, "state_tokens": 125, "latency_ms": 93.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 200}, "server_wall_seconds": 0.09058698100000129, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.71}}, "usage": {"input_tokens": 200, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.29203060269355774, "true": 0.7079693675041199}}, "native_timing": {"tokens": 200, "state_tokens": 144, "latency_ms": 89.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-14", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 217}, "server_wall_seconds": 0.08319523299999787, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.91}}, "usage": {"input_tokens": 217, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.09308559447526932, "true": 0.9069144129753113}}, "native_timing": {"tokens": 217, "state_tokens": 161, "latency_ms": 82.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-15", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 173}, "server_wall_seconds": 0.07997162699999905, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.74}}, "usage": {"input_tokens": 173, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.26388096809387207, "true": 0.7361190319061279}}, "native_timing": {"tokens": 173, "state_tokens": 117, "latency_ms": 78.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-judge_hard-18", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 140}, "server_wall_seconds": 0.07783569899999776, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.85}}, "usage": {"input_tokens": 140, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.15486544370651245, "true": 0.8451345562934875}}, "native_timing": {"tokens": 140, "state_tokens": 84, "latency_ms": 76.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-long_policy-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2706}, "server_wall_seconds": 0.25453284199999615, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "approve_all", "confidence": 0.34, "probabilities": {"approve_all": 0.51, "approve_lodging_only": 0.16, "approve_meals_only": 0.1, "reject_entire_claim": 0.23}}}, "usage": {"input_tokens": 2706, "output_tokens": 75}}, "unrounded_probabilities": {"decision": {"approve_all": 0.5087080001831055, "approve_lodging_only": 0.1629502773284912, "approve_meals_only": 0.09951456636190414, "reject_entire_claim": 0.22882714867591858}}, "native_timing": {"tokens": 2706, "state_tokens": 2615, "latency_ms": 249.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-long_policy-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2557}, "server_wall_seconds": 0.2417743399999992, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "deny_current_export", "confidence": 0.08, "probabilities": {"analytics_owner_only": 0.31, "deny_current_export": 0.31, "privacy_and_security": 0.21, "privacy_only": 0.17}}}, "usage": {"input_tokens": 2557, "output_tokens": 73}}, "unrounded_probabilities": {"decision": {"analytics_owner_only": 0.30910149216651917, "deny_current_export": 0.31250229477882385, "privacy_and_security": 0.209828719496727, "privacy_only": 0.1685674637556076}}, "native_timing": {"tokens": 2557, "state_tokens": 2470, "latency_ms": 235.9, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-long_policy-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2518}, "server_wall_seconds": 0.21498960899999986, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "delete_profile_omar_april_and_table", "confidence": 0.33, "probabilities": {"delete_everything": 0.24, "delete_nothing": 0.12, "delete_only_profile": 0.14, "delete_profile_omar_april_and_table": 0.5}}}, "usage": {"input_tokens": 2518, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"delete_everything": 0.2409096360206604, "delete_nothing": 0.11932789534330368, "delete_only_profile": 0.14341115951538086, "delete_profile_omar_april_and_table": 0.49635130167007446}}, "native_timing": {"tokens": 2518, "state_tokens": 2422, "latency_ms": 210.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-long_policy-06", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2602}, "server_wall_seconds": 0.2097587440000055, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "east_ward", "confidence": 0.2, "probabilities": {"east_ward": 0.4, "harbor_neighbors": 0.16, "no_award": 0.15, "river_arts": 0.3}}}, "usage": {"input_tokens": 2602, "output_tokens": 71}}, "unrounded_probabilities": {"decision": {"east_ward": 0.3996564447879791, "harbor_neighbors": 0.15601423382759094, "no_award": 0.1463586837053299, "river_arts": 0.29797062277793884}}, "native_timing": {"tokens": 2602, "state_tokens": 2517, "latency_ms": 205.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-routing_hard-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 274}, "server_wall_seconds": 0.09189306400000419, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "office_file_repair", "confidence": 0.92, "probabilities": {"data_pipeline": 0.02, "desktop_support": 0.01, "finance_review": 0.01, "office_file_repair": 0.93, "spreadsheet_formula": 0.02}}}, "usage": {"input_tokens": 274, "output_tokens": 84}}, "unrounded_probabilities": {"decision": {"data_pipeline": 0.019340617582201958, "desktop_support": 0.013968479819595814, "finance_review": 0.012922046706080437, "office_file_repair": 0.9340332746505737, "spreadsheet_formula": 0.019735587760806084}}, "native_timing": {"tokens": 274, "state_tokens": 155, "latency_ms": 90.0, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-routing_hard-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 255}, "server_wall_seconds": 0.09639520499999321, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "ledger_reconciliation", "confidence": 0.92, "probabilities": {"customer_support": 0.01, "fraud_review": 0.0, "ledger_reconciliation": 0.93, "mobile_app": 0.02, "payment_disputes": 0.03}}}, "usage": {"input_tokens": 255, "output_tokens": 83}}, "unrounded_probabilities": {"decision": {"customer_support": 0.009882713668048382, "fraud_review": 0.004633682314306498, "ledger_reconciliation": 0.9325408935546875, "mobile_app": 0.020784074440598488, "payment_disputes": 0.03215861693024635}}, "native_timing": {"tokens": 255, "state_tokens": 151, "latency_ms": 94.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-routing_hard-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 255}, "server_wall_seconds": 0.09797328700000207, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "authoritative_dns", "confidence": 0.52, "probabilities": {"api_platform": 0.16, "authoritative_dns": 0.62, "endpoint_devices": 0.04, "network_dns": 0.13, "tls_certificates": 0.05}}}, "usage": {"input_tokens": 255, "output_tokens": 82}}, "unrounded_probabilities": {"decision": {"api_platform": 0.16005395352840424, "authoritative_dns": 0.6193749904632568, "endpoint_devices": 0.042607229202985764, "network_dns": 0.1308065801858902, "tls_certificates": 0.04715719074010849}}, "native_timing": {"tokens": 255, "state_tokens": 153, "latency_ms": 96.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-routing_hard-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 238}, "server_wall_seconds": 0.09450517399999825, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "release_notarization", "confidence": 0.33, "probabilities": {"ci_infrastructure": 0.13, "code_signing": 0.24, "endpoint_security": 0.03, "macos_runtime": 0.13, "release_notarization": 0.46}}}, "usage": {"input_tokens": 238, "output_tokens": 86}}, "unrounded_probabilities": {"decision": {"ci_infrastructure": 0.13136199116706848, "code_signing": 0.23988661170005798, "endpoint_security": 0.032373808324337006, "macos_runtime": 0.1324843168258667, "release_notarization": 0.463893324136734}}, "native_timing": {"tokens": 238, "state_tokens": 139, "latency_ms": 92.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-routing_hard-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 244}, "server_wall_seconds": 0.08933141499999664, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "source_access", "confidence": 0.59, "probabilities": {"git_client": 0.07, "identity_login": 0.06, "repository_admin": 0.07, "source_access": 0.67, "vpn_network": 0.13}}}, "usage": {"input_tokens": 244, "output_tokens": 79}}, "unrounded_probabilities": {"decision": {"git_client": 0.06719546765089035, "identity_login": 0.055769484490156174, "repository_admin": 0.0717688649892807, "source_access": 0.672076404094696, "vpn_network": 0.13318975269794464}}, "native_timing": {"tokens": 244, "state_tokens": 142, "latency_ms": 87.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-temporal_numeric-01", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 251}, "server_wall_seconds": 0.10541156700000442, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "before_open", "confidence": 0.09, "probabilities": {"before_open": 0.31, "late_by_2h_or_more": 0.22, "late_by_under_2h": 0.15, "within_window": 0.31}}}, "usage": {"input_tokens": 251, "output_tokens": 78}}, "unrounded_probabilities": {"decision": {"before_open": 0.3149806559085846, "late_by_2h_or_more": 0.22145475447177887, "late_by_under_2h": 0.15250636637210846, "within_window": 0.31105825304985046}}, "native_timing": {"tokens": 251, "state_tokens": 160, "latency_ms": 102.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-temporal_numeric-02", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 265}, "server_wall_seconds": 0.09520338299999764, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "over_limit", "confidence": 0.19, "probabilities": {"at_least_100gb_below": 0.2, "exactly_equal": 0.16, "over_limit": 0.39, "under_100gb_below": 0.25}}}, "usage": {"input_tokens": 265, "output_tokens": 80}}, "unrounded_probabilities": {"decision": {"at_least_100gb_below": 0.19546040892601013, "exactly_equal": 0.15859277546405792, "over_limit": 0.39113572239875793, "under_100gb_below": 0.25481101870536804}}, "native_timing": {"tokens": 265, "state_tokens": 165, "latency_ms": 93.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-temporal_numeric-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 199}, "server_wall_seconds": 0.08604491000000536, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "12_credits", "confidence": 0.3, "probabilities": {"12_credits": 0.48, "15_credits": 0.27, "6_credits": 0.13, "9_credits": 0.12}}}, "usage": {"input_tokens": 199, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"12_credits": 0.4766136407852173, "15_credits": 0.2709546685218811, "6_credits": 0.12991055846214294, "9_credits": 0.12252113968133926}}, "native_timing": {"tokens": 199, "state_tokens": 113, "latency_ms": 84.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-b-temporal_numeric-04", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 214}, "server_wall_seconds": 0.08199892100000028, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "clear_fail", "confidence": 0.24, "probabilities": {"clear_fail": 0.5, "narrow_fail": 0.33, "pass": 0.17}}}, "usage": {"input_tokens": 214, "output_tokens": 58}}, "unrounded_probabilities": {"decision": {"clear_fail": 0.4951353073120117, "narrow_fail": 0.3327992260456085, "pass": 0.1720653921365738}}, "native_timing": {"tokens": 214, "state_tokens": 154, "latency_ms": 80.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-03", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 360}, "server_wall_seconds": 0.09097532199999847, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.87}}, "usage": {"input_tokens": 360, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.1278839111328125, "true": 0.8721160888671875}}, "native_timing": {"tokens": 360, "state_tokens": 293, "latency_ms": 89.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-05", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 322}, "server_wall_seconds": 0.0887340120000033, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.88}}, "usage": {"input_tokens": 322, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.12228310108184814, "true": 0.8777168989181519}}, "native_timing": {"tokens": 322, "state_tokens": 255, "latency_ms": 87.5, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 288}, "server_wall_seconds": 0.09899549600000057, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.95}}, "usage": {"input_tokens": 288, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.05045074224472046, "true": 0.9495492577552795}}, "native_timing": {"tokens": 288, "state_tokens": 221, "latency_ms": 97.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-08", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 281}, "server_wall_seconds": 0.099838179999999, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.91}}, "usage": {"input_tokens": 281, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.09148705750703812, "true": 0.9085129499435425}}, "native_timing": {"tokens": 281, "state_tokens": 214, "latency_ms": 98.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 311}, "server_wall_seconds": 0.09495617800000389, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.94}}, "usage": {"input_tokens": 311, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.055882155895233154, "true": 0.9441178441047668}}, "native_timing": {"tokens": 311, "state_tokens": 244, "latency_ms": 93.2, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-10", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 341}, "server_wall_seconds": 0.10393292899999551, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.91}}, "usage": {"input_tokens": 341, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.09264988452196121, "true": 0.9073501229286194}}, "native_timing": {"tokens": 341, "state_tokens": 274, "latency_ms": 102.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 309}, "server_wall_seconds": 0.09836062999999484, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.83}}, "usage": {"input_tokens": 309, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.1708286702632904, "true": 0.829171359539032}}, "native_timing": {"tokens": 309, "state_tokens": 242, "latency_ms": 96.6, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-13", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 301}, "server_wall_seconds": 0.100161417999999, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.91}}, "usage": {"input_tokens": 301, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.09432859718799591, "true": 0.9056713581085205}}, "native_timing": {"tokens": 301, "state_tokens": 234, "latency_ms": 98.7, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-judge_hard-15", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 254}, "server_wall_seconds": 0.08564921800000036, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "noul", "noul": 0.73}}, "usage": {"input_tokens": 254, "output_tokens": 20}}, "unrounded_probabilities": {"decision": {"false": 0.27382248640060425, "true": 0.7261775135993958}}, "native_timing": {"tokens": 254, "state_tokens": 187, "latency_ms": 84.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-multi_hop-07", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2495}, "server_wall_seconds": 0.2002428439999946, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "programme_review", "confidence": 0.14, "probabilities": {"partial_release": 0.17, "programme_review": 0.36, "reject_draw": 0.22, "release_draw": 0.25}}}, "usage": {"input_tokens": 2495, "output_tokens": 69}}, "unrounded_probabilities": {"decision": {"partial_release": 0.16899947822093964, "programme_review": 0.35868072509765625, "reject_draw": 0.22346027195453644, "release_draw": 0.24885950982570648}}, "native_timing": {"tokens": 2495, "state_tokens": 2419, "latency_ms": 196.1, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-multi_hop-09", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2514}, "server_wall_seconds": 0.20834305099999995, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "bus_substitution", "confidence": 0.1, "probabilities": {"bus_substitution": 0.32, "call_fen_cross": 0.15, "pass_fen_cross": 0.24, "terminate_oakmere": 0.29}}}, "usage": {"input_tokens": 2514, "output_tokens": 76}}, "unrounded_probabilities": {"decision": {"bus_substitution": 0.32310542464256287, "call_fen_cross": 0.14587731659412384, "pass_fen_cross": 0.24387302994728088, "terminate_oakmere": 0.28714415431022644}}, "native_timing": {"tokens": 2514, "state_tokens": 2428, "latency_ms": 203.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-multi_hop-11", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2592}, "server_wall_seconds": 0.24520428100000657, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "coverage_excluded", "confidence": 0.38, "probabilities": {"adjuster_referral": 0.06, "coverage_confirmed": 0.17, "coverage_excluded": 0.54, "documentation_needed": 0.23}}}, "usage": {"input_tokens": 2592, "output_tokens": 74}}, "unrounded_probabilities": {"decision": {"adjuster_referral": 0.06468750536441803, "coverage_confirmed": 0.1712184101343155, "coverage_excluded": 0.5364008545875549, "documentation_needed": 0.22769322991371155}}, "native_timing": {"tokens": 2592, "state_tokens": 2509, "latency_ms": 240.4, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-multi_hop-12", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 513}, "server_wall_seconds": 0.11820583200000101, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "deny_request", "confidence": 0.12, "probabilities": {"conservation_hold": 0.22, "deny_request": 0.34, "digital_copy_only": 0.23, "supervised_delivery": 0.21}}}, "usage": {"input_tokens": 513, "output_tokens": 72}}, "unrounded_probabilities": {"decision": {"conservation_hold": 0.21583281457424164, "deny_request": 0.33684253692626953, "digital_copy_only": 0.23492424190044403, "supervised_delivery": 0.21240034699440002}}, "native_timing": {"tokens": 513, "state_tokens": 431, "latency_ms": 115.8, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}
{"id": "benchmark:hard-sol-c-multi_hop-13", "runtime": {"model": "Kev-0.8B", "checkpoint_revision": "54f4f8777356cd5bbbb6c6919c657f26e6f2f6d8", "source_commit": "5e94a28818cfd3d0ec9b8bca046dc8db0d79a704", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.17.0", "numpy": "2.5.3", "safetensors": "0.8.0"}, "checkpoint_hashes": {"head.pt": "39f4343ccccc65e583bbfff0de0e11bfedb849fcfaaf94b50ac4f2b73bc79c65", "adapter_model.safetensors": "c81d5716f0af7622d8d2b97013c333d48263ca01113f9a7cf4526e96f6ac0b26"}, "base_revision": "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68", "base_sha256": "c2b1e5a17d9c1e27685d92ed9b382911ebb99955ecd89052d1721241adfbab6c", "dtype": "bfloat16", "adapter_merged": false, "temperature": 2.406050072164233, "max_state": 8192, "max_branch": 8192, "prefix_cache": false, "date_facts": false, "probability_precision": "native API,2 decimal places; unrounded retained as diagnostic"}, "input_audit": {"state_truncated": false, "encoded_tokens": 2564}, "server_wall_seconds": 0.21391091899999992, "ok": true, "response": {"model": "kev-0.8b", "answers": {"decision": {"type": "choice", "choice": "normal_release", "confidence": 0.43, "probabilities": {"engineering_review": 0.23, "normal_release": 0.58, "rework": 0.12, "scrap_lot": 0.08}}}, "usage": {"input_tokens": 2564, "output_tokens": 70}}, "unrounded_probabilities": {"decision": {"engineering_review": 0.22581230103969574, "normal_release": 0.5755835771560669, "rework": 0.11615115404129028, "scrap_lot": 0.08245296776294708}}, "native_timing": {"tokens": 2564, "state_tokens": 2487, "latency_ms": 209.3, "prefix_cache_hit": false}, "gpu_peak_allocated_bytes": 2083974656}