{"id": "benchmark:original-policy-01-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 28, "state_tokens_kept": 28, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.7892645380000012, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.2462, "confidence": 0.7538, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 81, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2436223488}
{"id": "benchmark:original-policy-01-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 30, "state_tokens_kept": 30, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.030241075999999367, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.4833, "confidence": 0.5167, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 83, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-02-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 20, "state_tokens_kept": 20, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026220935000001333, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6533, "confidence": 0.6533, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 73, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-02-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 19, "state_tokens_kept": 19, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026285706000003017, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7563, "confidence": 0.7563, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 72, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-03-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 22, "state_tokens_kept": 22, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026913880000002166, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.251, "confidence": 0.749, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 75, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-03-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 22, "state_tokens_kept": 22, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02680905800000133, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7049, "confidence": 0.7049, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 75, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-04-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 30, "state_tokens_kept": 30, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027591596000000607, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5039, "confidence": 0.5039, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 83, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-04-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 28, "state_tokens_kept": 28, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026076551999999253, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.4042, "confidence": 0.5958, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 81, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441474560}
{"id": "benchmark:original-policy-05-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 33, "state_tokens_kept": 33, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02645585000000139, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5089, "confidence": 0.5089, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 86, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-policy-05-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 26, "state_tokens_kept": 26, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025697842999999665, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.1997, "confidence": 0.8003, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 79, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-policy-06-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 31, "state_tokens_kept": 31, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02538820499999872, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8252, "confidence": 0.8252, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 84, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-policy-06-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 21, "state_tokens_kept": 21, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02522682200000048, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6505, "confidence": 0.6505, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-01-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03826521799999938, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.0378, "change_address": 0.0041, "other": 0.0101, "refund": 0.9397, "status": 0.0084}, "confidence": 0.8191, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-01-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027457032000000936, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.0531, "change_address": 0.0065, "other": 0.0097, "refund": 0.9246, "status": 0.006}, "confidence": 0.7906, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 76, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-02-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025964109000000235, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "change_address", "probabilities": {"cancel": 0.0277, "change_address": 0.5561, "other": 0.2603, "refund": 0.076, "status": 0.0799}, "confidence": 0.2707, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 75, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-02-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025324833999999186, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "status", "probabilities": {"cancel": 0.0144, "change_address": 0.1223, "other": 0.1177, "refund": 0.0342, "status": 0.7114}, "confidence": 0.4236, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-03-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02487511400000031, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "status", "probabilities": {"cancel": 0.0214, "change_address": 0.3214, "other": 0.0813, "refund": 0.0794, "status": 0.4965}, "confidence": 0.2545, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 76, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-03-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024936645000000368, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "change_address", "probabilities": {"cancel": 0.0037, "change_address": 0.9605, "other": 0.0116, "refund": 0.0118, "status": 0.0125}, "confidence": 0.8647, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-04-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 17, "state_tokens_kept": 17, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025271352999997276, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "cancel", "probabilities": {"cancel": 0.7731, "change_address": 0.0113, "other": 0.082, "refund": 0.1065, "status": 0.0271}, "confidence": 0.5085, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 79, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-04-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025229601999999574, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "cancel", "probabilities": {"cancel": 0.7725, "change_address": 0.0176, "other": 0.0893, "refund": 0.0903, "status": 0.0303}, "confidence": 0.4972, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-05-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02512618999999816, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.0128, "change_address": 0.0161, "other": 0.0954, "refund": 0.8241, "status": 0.0515}, "confidence": 0.5906, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-05-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024942376000002042, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "refund", "probabilities": {"cancel": 0.0133, "change_address": 0.009, "other": 0.0516, "refund": 0.9078, "status": 0.0184}, "confidence": 0.7428, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 74, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-06-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026677304999999762, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "status", "probabilities": {"cancel": 0.0354, "change_address": 0.0073, "other": 0.0738, "refund": 0.028, "status": 0.8554}, "confidence": 0.6393, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 72, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-intent-06-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02696434100000289, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "status", "probabilities": {"cancel": 0.3735, "change_address": 0.0068, "other": 0.0371, "refund": 0.0158, "status": 0.5669}, "confidence": 0.4339, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 77, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441486848}
{"id": "benchmark:original-ordinal-01-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027041312999998013, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 0.967, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.4516, "1": 0.1809, "2": 0.3164, "3": 0.0511}, "confidence": 0.1456, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 85, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441490944}
{"id": "benchmark:original-ordinal-01-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025736234000000024, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 0.9687, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.356, "1": 0.3429, "2": 0.2773, "3": 0.0237}, "confidence": 0.1495, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 87, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441499136}
{"id": "benchmark:original-ordinal-02-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02541539699999973, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.0185, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.0442, "1": 0.9055, "2": 0.0379, "3": 0.0124}, "confidence": 0.707, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 89, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441507328}
{"id": "benchmark:original-ordinal-02-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02784062100000284, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 0.9302, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.1302, "1": 0.8291, "2": 0.021, "3": 0.0197}, "confidence": 0.5821, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 90, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441511424}
{"id": "benchmark:original-ordinal-03-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02623885500000256, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.9614, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.0177, "1": 0.046, "2": 0.8934, "3": 0.0429}, "confidence": 0.6762, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 86, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441511424}
{"id": "benchmark:original-ordinal-03-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02553145999999984, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 2.0013, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.034, "1": 0.0972, "2": 0.7022, "3": 0.1665}, "confidence": 0.3592, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 88, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441511424}
{"id": "benchmark:original-ordinal-04-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025191791000001018, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 2.9149, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.0127, "1": 0.0128, "2": 0.0215, "3": 0.953}, "confidence": 0.8272, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 88, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441511424}
{"id": "benchmark:original-ordinal-04-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025422357000000062, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 2.801, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.0225, "1": 0.0362, "2": 0.0592, "3": 0.8822}, "confidence": 0.6513, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 87, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441511424}
{"id": "benchmark:original-ordinal-05-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 16, "state_tokens_kept": 16, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027327360000001022, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.0075, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.4642, "1": 0.2329, "2": 0.134, "3": 0.1689}, "confidence": 0.0873, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 91, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441515520}
{"id": "benchmark:original-ordinal-05-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02585756599999911, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.3764, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.2202, "1": 0.2962, "2": 0.3708, "3": 0.1129}, "confidence": 0.0567, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 90, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441515520}
{"id": "benchmark:original-ordinal-06-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 17, "state_tokens_kept": 17, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026006879000000538, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.7424, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.0681, "1": 0.1561, "2": 0.741, "3": 0.0348}, "confidence": 0.4143, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 92, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441519616}
{"id": "benchmark:original-ordinal-06-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 24, "state_tokens_kept": 24, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.04385976600000063, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.9235, "legend": {"0": "No function impaired; cosmetic only", "1": "One user or a nonessential function impaired, with a workaround", "2": "Many users blocked from a core function, no data loss", "3": "Confirmed irreversible data loss or physical harm"}, "probabilities": {"0": 0.0137, "1": 0.085, "2": 0.8655, "3": 0.0359}, "confidence": 0.6302, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 99, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-01-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027193887999999333, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "pickup", "probabilities": {"courier": 0.1244, "pickup": 0.7725, "post": 0.0373, "unknown": 0.0658}, "confidence": 0.4515, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 68, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-01-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026228926000001707, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "pickup", "probabilities": {"courier": 0.1783, "pickup": 0.4692, "post": 0.0519, "unknown": 0.3007}, "confidence": 0.1507, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 70, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-02-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025971608999999063, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "unknown", "probabilities": {"courier": 0.0912, "pickup": 0.0268, "post": 0.2508, "unknown": 0.6313}, "confidence": 0.3129, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 69, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-02-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02566139199999995, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "post", "probabilities": {"courier": 0.0571, "pickup": 0.0159, "post": 0.7293, "unknown": 0.1978}, "confidence": 0.4373, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 69, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-03-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 18, "state_tokens_kept": 18, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025779785000001, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "pickup", "probabilities": {"courier": 0.3076, "pickup": 0.5829, "post": 0.0369, "unknown": 0.0726}, "confidence": 0.2863, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 73, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-03-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 17, "state_tokens_kept": 17, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024545936999999185, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.8675, "pickup": 0.0301, "post": 0.0271, "unknown": 0.0753}, "confidence": 0.624, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 72, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-04-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025848035999999297, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "post", "probabilities": {"courier": 0.0767, "pickup": 0.0944, "post": 0.7691, "unknown": 0.0597}, "confidence": 0.4301, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 69, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-04-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025675673000002064, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "post", "probabilities": {"courier": 0.0622, "pickup": 0.0163, "post": 0.8534, "unknown": 0.0681}, "confidence": 0.5974, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 68, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-05-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02532535499999966, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.5853, "pickup": 0.0223, "post": 0.0548, "unknown": 0.3376}, "confidence": 0.3334, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 70, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-05-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025903028000001882, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.5331, "pickup": 0.0203, "post": 0.1046, "unknown": 0.342}, "confidence": 0.266, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 66, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-06-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02523581299999833, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.8788, "pickup": 0.0083, "post": 0.0807, "unknown": 0.0322}, "confidence": 0.6631, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 70, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-extraction-06-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025408816000002332, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "courier", "probabilities": {"courier": 0.6254, "pickup": 0.0159, "post": 0.2656, "unknown": 0.0932}, "confidence": 0.3274, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 69, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-01-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025466158000000405, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.1216, "confidence": 0.8784, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 61, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-01-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025320045000000846, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5285, "confidence": 0.5285, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 58, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-02-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 18, "state_tokens_kept": 18, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02564334200000218, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7828, "confidence": 0.7828, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 65, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-02-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 19, "state_tokens_kept": 19, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024948497000000458, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3653, "confidence": 0.6347, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 66, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-03-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 18, "state_tokens_kept": 18, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02544237700000096, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8138, "confidence": 0.8138, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 65, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-03-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 21, "state_tokens_kept": 21, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025069419000001147, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3501, "confidence": 0.6499, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 68, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-04-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 24, "state_tokens_kept": 24, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024993827999999496, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.9261, "confidence": 0.9261, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 71, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-04-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 21, "state_tokens_kept": 21, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024554928000000587, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.92, "confidence": 0.92, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 68, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-05-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02518199100000018, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.2284, "confidence": 0.7716, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 59, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-05-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025265183000001912, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.4147, "confidence": 0.5853, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 60, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-06-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 24, "state_tokens_kept": 24, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024728072000002044, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7933, "confidence": 0.7933, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 71, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-adequacy-06-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 24, "state_tokens_kept": 24, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024682210999998233, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3003, "confidence": 0.6997, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 71, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441548288}
{"id": "benchmark:original-routing-01-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026457999999998094, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "math", "probabilities": {"coding": 0.1681, "coding_agent": 0.0235, "document": 0.0039, "general": 0.0318, "math": 0.7534, "tools": 0.0192}, "confidence": 0.5488, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 105, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441581056}
{"id": "benchmark:original-routing-01-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 8, "state_tokens_kept": 8, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025347165000003002, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "math", "probabilities": {"coding": 0.1438, "coding_agent": 0.0307, "document": 0.034, "general": 0.1342, "math": 0.5971, "tools": 0.0603}, "confidence": 0.3039, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 102, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441581056}
{"id": "benchmark:original-routing-02-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02595637900000014, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "coding", "probabilities": {"coding": 0.9748, "coding_agent": 0.0129, "document": 0.0006, "general": 0.004, "math": 0.0006, "tools": 0.007}, "confidence": 0.918, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 109, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441597440}
{"id": "benchmark:original-routing-02-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 9, "state_tokens_kept": 9, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025384276000000483, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "coding", "probabilities": {"coding": 0.9737, "coding_agent": 0.0072, "document": 0.0021, "general": 0.0065, "math": 0.0014, "tools": 0.0091}, "confidence": 0.9112, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 103, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441597440}
{"id": "benchmark:original-routing-03-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.0263476570000023, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "coding", "probabilities": {"coding": 0.5537, "coding_agent": 0.2873, "document": 0.0046, "general": 0.0726, "math": 0.0031, "tools": 0.0788}, "confidence": 0.3755, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 108, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441597440}
{"id": "benchmark:original-routing-03-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 17, "state_tokens_kept": 17, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025770125000001087, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "coding", "probabilities": {"coding": 0.4809, "coding_agent": 0.3925, "document": 0.0071, "general": 0.0843, "math": 0.0046, "tools": 0.0305}, "confidence": 0.3894, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 111, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:original-routing-04-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025291843999998065, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.0103, "coding_agent": 0.0055, "document": 0.0237, "general": 0.9582, "math": 0.0012, "tools": 0.0012}, "confidence": 0.8764, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 104, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:original-routing-04-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025266373999997427, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.1526, "coding_agent": 0.0609, "document": 0.0559, "general": 0.6442, "math": 0.0272, "tools": 0.0592}, "confidence": 0.3486, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 104, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:original-routing-05-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025831796999998602, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.2006, "coding_agent": 0.0403, "document": 0.0538, "general": 0.6451, "math": 0.0116, "tools": 0.0486}, "confidence": 0.3914, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 105, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:original-routing-05-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025695273000000185, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.2445, "coding_agent": 0.0286, "document": 0.0406, "general": 0.6305, "math": 0.0126, "tools": 0.0432}, "confidence": 0.4096, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 106, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:original-routing-06-0", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024975566999998478, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.2884, "coding_agent": 0.0999, "document": 0.0356, "general": 0.5072, "math": 0.0133, "tools": 0.0556}, "confidence": 0.2913, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 104, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:original-routing-06-1", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025011488000000526, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "general", "probabilities": {"coding": 0.091, "coding_agent": 0.0209, "document": 0.0047, "general": 0.8616, "math": 0.0042, "tools": 0.0176}, "confidence": 0.6949, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 104, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-00", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 17, "state_tokens_kept": 17, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025540879000001127, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "track_order", "probabilities": {"billing_question": 0.0056, "cancel_order": 0.0044, "change_address": 0.0039, "report_damage": 0.0472, "track_order": 0.939}, "confidence": 0.8275, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 102, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025525878999999918, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "cancel_order", "probabilities": {"billing_question": 0.0007, "cancel_order": 0.9941, "change_address": 0.0008, "report_damage": 0.0036, "track_order": 0.0009}, "confidence": 0.9733, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 100, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 19, "state_tokens_kept": 19, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025204752000000497, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "change_address", "probabilities": {"billing_question": 0.0033, "cancel_order": 0.0048, "change_address": 0.9812, "report_damage": 0.007, "track_order": 0.0038}, "confidence": 0.9262, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 104, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 9, "state_tokens_kept": 9, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026020769999998805, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "report_damage", "probabilities": {"billing_question": 0.0037, "cancel_order": 0.0042, "change_address": 0.0039, "report_damage": 0.9801, "track_order": 0.008}, "confidence": 0.9229, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 94, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026262995999999816, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "billing_question", "probabilities": {"billing_question": 0.7383, "cancel_order": 0.0265, "change_address": 0.0179, "report_damage": 0.1147, "track_order": 0.1027}, "confidence": 0.4568, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 99, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026613223999998326, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "track_order", "probabilities": {"billing_question": 0.0086, "cancel_order": 0.0073, "change_address": 0.0061, "report_damage": 0.0524, "track_order": 0.9255}, "confidence": 0.7923, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 98, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02537292600000285, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "set_alarm", "probabilities": {"play_music": 0.0007, "send_message": 0.0007, "set_alarm": 0.9972, "turn_off_lights": 0.0008, "weather": 0.0007}, "confidence": 0.9857, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 86, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 5, "state_tokens_kept": 5, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02480582199999759, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "play_music", "probabilities": {"play_music": 0.9848, "send_message": 0.0032, "set_alarm": 0.0042, "turn_off_lights": 0.0036, "weather": 0.0042}, "confidence": 0.9381, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 80, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 7, "state_tokens_kept": 7, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025327234999998893, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "weather", "probabilities": {"play_music": 0.0069, "send_message": 0.0072, "set_alarm": 0.0071, "turn_off_lights": 0.0082, "weather": 0.9706}, "confidence": 0.8924, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 82, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025381915999997062, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "send_message", "probabilities": {"play_music": 0.002, "send_message": 0.9916, "set_alarm": 0.0021, "turn_off_lights": 0.0022, "weather": 0.0021}, "confidence": 0.9627, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 85, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 8, "state_tokens_kept": 8, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02528718399999974, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "turn_off_lights", "probabilities": {"play_music": 0.0019, "send_message": 0.0019, "set_alarm": 0.0094, "turn_off_lights": 0.9846, "weather": 0.0023}, "confidence": 0.9399, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 83, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-intent-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 7, "state_tokens_kept": 7, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02541972699999917, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "set_alarm", "probabilities": {"play_music": 0.0002, "send_message": 0.0002, "set_alarm": 0.9991, "turn_off_lights": 0.0002, "weather": 0.0002}, "confidence": 0.9947, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 82, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-00", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 18, "state_tokens_kept": 18, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025508319000000057, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.9001, "confidence": 0.9001, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 63, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024963706000001196, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7385, "confidence": 0.7385, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 60, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 20, "state_tokens_kept": 20, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024763961999997974, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6999, "confidence": 0.6999, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 64, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 24, "state_tokens_kept": 24, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025121750000000276, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.1027, "confidence": 0.8973, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 68, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 17, "state_tokens_kept": 17, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02465356000000085, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8189, "confidence": 0.8189, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 64, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 18, "state_tokens_kept": 18, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02503544800000057, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.2968, "confidence": 0.7032, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 65, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 19, "state_tokens_kept": 19, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02444326399999852, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.4479, "confidence": 0.5521, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 72, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025048747999999676, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.1423, "confidence": 0.8577, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 60, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024837192999999758, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8357, "confidence": 0.8357, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 56, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024571776999998463, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.4519, "confidence": 0.5481, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 56, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 14, "state_tokens_kept": 14, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025091698999997192, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8737, "confidence": 0.8737, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 63, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-fact-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 15, "state_tokens_kept": 15, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02455068699999785, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.1132, "confidence": 0.8868, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 64, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-00", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 7, "state_tokens_kept": 7, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025218112000001014, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "paypal", "probabilities": {"bank_transfer": 0.0527, "cash": 0.0523, "credit_card": 0.2998, "paypal": 0.5952}, "confidence": 0.2935, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 54, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02541510699999705, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "cash", "probabilities": {"bank_transfer": 0.0378, "cash": 0.9321, "credit_card": 0.0176, "paypal": 0.0125}, "confidence": 0.7726, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 57, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02518758200000093, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "bank_transfer", "probabilities": {"bank_transfer": 0.99, "cash": 0.0024, "credit_card": 0.0044, "paypal": 0.0032}, "confidence": 0.9517, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 57, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 19, "state_tokens_kept": 19, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02580105499999874, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "high", "probabilities": {"high": 0.866, "low": 0.0094, "medium": 0.0147, "urgent": 0.1099}, "confidence": 0.6587, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 55, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 16, "state_tokens_kept": 16, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025565439999997608, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "low", "probabilities": {"high": 0.0043, "low": 0.96, "medium": 0.0104, "urgent": 0.0253}, "confidence": 0.8534, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 52, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 16, "state_tokens_kept": 16, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025408025999997363, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "urgent", "probabilities": {"high": 0.0182, "low": 0.0123, "medium": 0.0172, "urgent": 0.9523}, "confidence": 0.8244, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 52, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025522068999997316, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "M", "probabilities": {"L": 0.0216, "M": 0.9247, "S": 0.0356, "XL": 0.0181}, "confidence": 0.7499, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 45, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025396607000001126, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "XL", "probabilities": {"L": 0.0992, "M": 0.0154, "S": 0.0082, "XL": 0.8771}, "confidence": 0.6767, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 43, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 8, "state_tokens_kept": 8, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02521171200000083, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "thursday", "probabilities": {"friday": 0.0022, "monday": 0.0026, "thursday": 0.9891, "tuesday": 0.0029, "wednesday": 0.0032}, "confidence": 0.9532, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 50, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025440245999998723, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "monday", "probabilities": {"friday": 0.003, "monday": 0.987, "thursday": 0.0028, "tuesday": 0.0039, "wednesday": 0.0033}, "confidence": 0.9458, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 54, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 8, "state_tokens_kept": 8, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02500789699999828, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "friday", "probabilities": {"friday": 0.9305, "monday": 0.0185, "thursday": 0.0149, "tuesday": 0.0189, "wednesday": 0.0173}, "confidence": 0.7835, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 50, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-extraction-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025363016000000016, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "credit_card", "probabilities": {"bank_transfer": 0.0206, "cash": 0.0121, "credit_card": 0.9449, "paypal": 0.0224}, "confidence": 0.8038, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 57, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-00", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 10, "state_tokens_kept": 10, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02511047100000141, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "get_weather", "probabilities": {"convert_currency": 0.0003, "create_calendar_event": 0.0004, "get_weather": 0.9985, "send_email": 0.0004, "translate_text": 0.0004}, "confidence": 0.9918, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 90, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025927749999997474, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "send_email", "probabilities": {"convert_currency": 0.0008, "create_calendar_event": 0.0012, "get_weather": 0.0008, "send_email": 0.9962, "translate_text": 0.001}, "confidence": 0.9812, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 91, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02689774199999917, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "create_calendar_event", "probabilities": {"convert_currency": 0.0014, "create_calendar_event": 0.9941, "get_weather": 0.0014, "send_email": 0.0016, "translate_text": 0.0016}, "confidence": 0.9723, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 93, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 9, "state_tokens_kept": 9, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025664912999999956, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "convert_currency", "probabilities": {"convert_currency": 0.9998, "create_calendar_event": 0.0, "get_weather": 0.0, "send_email": 0.0, "translate_text": 0.0001}, "confidence": 0.9987, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 89, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 9, "state_tokens_kept": 9, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02538428699999784, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "translate_text", "probabilities": {"convert_currency": 0.0067, "create_calendar_event": 0.0131, "get_weather": 0.0069, "send_email": 0.009, "translate_text": 0.9642}, "confidence": 0.8742, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 89, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 12, "state_tokens_kept": 12, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025411328000000566, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "create_calendar_event", "probabilities": {"convert_currency": 0.0001, "create_calendar_event": 0.9998, "get_weather": 0.0001, "send_email": 0.0001, "translate_text": 0.0001}, "confidence": 0.9985, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 92, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024762803000001554, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "search_flights", "probabilities": {"book_restaurant": 0.0001, "call_taxi": 0.0001, "get_stock_price": 0.0001, "search_flights": 0.9998, "set_reminder": 0.0001}, "confidence": 0.9984, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 96, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 13, "state_tokens_kept": 13, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02537890700000034, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "book_restaurant", "probabilities": {"book_restaurant": 0.9999, "call_taxi": 0.0, "get_stock_price": 0.0, "search_flights": 0.0, "set_reminder": 0.0}, "confidence": 0.9991, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 98, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.024587858999996826, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "set_reminder", "probabilities": {"book_restaurant": 0.0092, "call_taxi": 0.0109, "get_stock_price": 0.0104, "search_flights": 0.0114, "set_reminder": 0.958}, "confidence": 0.8556, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 96, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 9, "state_tokens_kept": 9, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025103450999999666, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "get_stock_price", "probabilities": {"book_restaurant": 0.0, "call_taxi": 0.0, "get_stock_price": 0.9999, "search_flights": 0.0, "set_reminder": 0.0}, "confidence": 0.9993, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 94, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 9, "state_tokens_kept": 9, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025037519000001396, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "call_taxi", "probabilities": {"book_restaurant": 0.0022, "call_taxi": 0.9906, "get_stock_price": 0.0022, "search_flights": 0.0025, "set_reminder": 0.0025}, "confidence": 0.9588, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 94, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:easy-tool_selection-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 11, "state_tokens_kept": 11, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02448325699999998, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "set_reminder", "probabilities": {"book_restaurant": 0.0041, "call_taxi": 0.0045, "get_stock_price": 0.0047, "search_flights": 0.0048, "set_reminder": 0.9819}, "confidence": 0.9281, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 96, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2441605632}
{"id": "benchmark:hard-opus-a-long_policy-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 3514, "state_tokens_kept": 316, "state_truncated": true, "instruction_truncated": true, "options_truncated": true}, "server_wall_seconds": 0.03277209499999856, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "deny_repeated_seepage", "probabilities": {"deny_repeated_seepage": 0.455, "deny_vacancy_exclusion": 0.1251, "pay_full_estimate_less_deductible": 0.2555, "pay_subject_to_10000_sublimit": 0.0792, "pay_subject_to_15000_sublimit": 0.0852}, "confidence": 0.1441, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2319, "state_tokens_kept": 361, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.030789860000002278, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "budget_holder", "probabilities": {"budget_holder": 0.2841, "cfo": 0.2698, "department_head": 0.1701, "executive_committee": 0.1291, "vp_and_finance_director": 0.1469}, "confidence": 0.0318, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2165, "state_tokens_kept": 373, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029141971999997907, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "full_repair_no_charge", "probabilities": {"full_repair_no_charge": 0.3659, "not_covered": 0.1188, "parts_covered_labour_charged": 0.326, "pro_rata_replacement_credit": 0.1893}, "confidence": 0.0612, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2280, "state_tokens_kept": 380, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029008579999999284, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "license_exception_lvs", "probabilities": {"license_exception_lvs": 0.5289, "license_required": 0.0219, "no_license_required": 0.2236, "shipment_prohibited": 0.2256}, "confidence": 0.2126, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2052, "state_tokens_kept": 408, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027936055000001403, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5118, "confidence": 0.5118, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-13", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2198, "state_tokens_kept": 420, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027766301000003324, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.666, "confidence": 0.666, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-17", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2173, "state_tokens_kept": 387, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028487107000000123, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "security_incident_response", "probabilities": {"database_oncall": 0.1776, "no_page_ticket_only": 0.0699, "platform_oncall": 0.164, "security_incident_response": 0.4952, "vendor_support": 0.0933}, "confidence": 0.1557, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-long_policy-19", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2038, "state_tokens_kept": 434, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028534927999999127, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6162, "confidence": 0.6162, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-probability-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 320, "state_tokens_kept": 320, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025646512999998095, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6534, "confidence": 0.6534, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 386, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-probability-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 845, "state_tokens_kept": 447, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02645791200000147, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "early", "probabilities": {"early": 0.4623, "late": 0.2837, "on_time": 0.2539}, "confidence": 0.0332, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-probability-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 897, "state_tokens_kept": 446, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026027971999997845, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "escalated", "probabilities": {"escalated": 0.4773, "reopened": 0.2741, "resolved_first_contact": 0.2486}, "confidence": 0.0408, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-probability-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 272, "state_tokens_kept": 272, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027672138999999873, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5378, "confidence": 0.5378, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 322, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2448718336}
{"id": "benchmark:hard-opus-a-temporal_numeric-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 363, "state_tokens_kept": 363, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026784059000000582, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6374, "confidence": 0.6374, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 427, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449288704}
{"id": "benchmark:hard-opus-a-temporal_numeric-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 353, "state_tokens_kept": 353, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02633546899999928, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8582, "confidence": 0.8582, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 428, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449298944}
{"id": "benchmark:hard-opus-a-temporal_numeric-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 843, "state_tokens_kept": 435, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027120307000000565, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "sep_26", "probabilities": {"sep_24": 0.2088, "sep_25": 0.2116, "sep_26": 0.2599, "sep_27": 0.1741, "sep_28": 0.1456}, "confidence": 0.0116, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449298944}
{"id": "benchmark:hard-opus-a-temporal_numeric-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 375, "state_tokens_kept": 375, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02777712199999982, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8724, "confidence": 0.8724, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 449, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449298944}
{"id": "benchmark:hard-opus-a-temporal_numeric-12", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 406, "state_tokens_kept": 406, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02831969400000034, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.8455, "legend": {"0": "No violations", "1": "Exactly one violation", "2": "Exactly two violations", "3": "Three or more violations"}, "probabilities": {"0": 0.0677, "1": 0.1797, "2": 0.592, "3": 0.1606}, "confidence": 0.2103, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 476, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449298944}
{"id": "benchmark:hard-opus-b-ambiguous-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 410, "state_tokens_kept": 410, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028044927999999913, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "real_incident", "probabilities": {"cannot_determine": 0.1381, "false_positive": 0.3206, "real_incident": 0.5413}, "confidence": 0.1167, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 494, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-ambiguous-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 353, "state_tokens_kept": 353, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02960864399999963, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "within_limitation", "probabilities": {"insufficient_information": 0.2149, "time_barred": 0.2488, "within_limitation": 0.5363}, "confidence": 0.08, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 449, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-ambiguous-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 369, "state_tokens_kept": 369, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027622877999998963, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "over_limit", "probabilities": {"insufficient_information": 0.1061, "over_limit": 0.5984, "within_policy": 0.2955}, "confidence": 0.1757, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 447, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-ambiguous-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 518, "state_tokens_kept": 435, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027508765000000324, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "compromised", "probabilities": {"compromised": 0.4397, "inconclusive": 0.2622, "legitimate": 0.2982}, "confidence": 0.0232, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-ambiguous-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 377, "state_tokens_kept": 377, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027268798999998012, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "send_to_alex_moreno", "probabilities": {"ask_user_which_alex": 0.3053, "send_to_alex_moreno": 0.4166, "send_to_alexandra_lind": 0.2781}, "confidence": 0.0143, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 456, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-ambiguous-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 324, "state_tokens_kept": 324, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02769615900000133, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "grade_should_be_higher", "probabilities": {"cannot_determine": 0.2084, "grade_correct": 0.3673, "grade_should_be_higher": 0.4243}, "confidence": 0.0366, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 406, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-ambiguous-13", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 423, "state_tokens_kept": 399, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027459832999998213, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "split_50_50", "probabilities": {"cannot_determine": 0.1352, "charge_data_team": 0.2561, "charge_platform_team": 0.245, "split_50_50": 0.3637}, "confidence": 0.0393, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-multi_hop-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2617, "state_tokens_kept": 422, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029830976999999592, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "tier3_cfo", "probabilities": {"tier1_team_lead": 0.2481, "tier2_department_head": 0.211, "tier3_cfo": 0.2906, "tier4_board": 0.2502}, "confidence": 0.0046, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-multi_hop-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2124, "state_tokens_kept": 421, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03068285699999862, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "no_page_ticket_only", "probabilities": {"ana": 0.016, "bjorn": 0.0097, "chen": 0.0081, "dmitri": 0.0095, "esra": 0.0722, "no_page_ticket_only": 0.8845}, "confidence": 0.725, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-multi_hop-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2224, "state_tokens_kept": 426, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02978084599999775, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.9203, "legend": {"0": "Level 0 — no award", "1": "Level 1 — EUR 500", "2": "Level 2 — EUR 1,200", "3": "Level 3 — EUR 2,000", "4": "Level 4 — EUR 3,000"}, "probabilities": {"0": 0.1241, "1": 0.2435, "2": 0.3389, "3": 0.1749, "4": 0.1186}, "confidence": 0.051, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-multi_hop-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2150, "state_tokens_kept": 389, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029800616999999363, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "credit_10_percent", "probabilities": {"credit_10_percent": 0.2069, "credit_20_percent": 0.1596, "credit_25_percent": 0.1335, "credit_50_percent": 0.1589, "credit_full_month": 0.1866, "no_credit": 0.1546}, "confidence": 0.0056, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-multi_hop-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 1942, "state_tokens_kept": 423, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029117411000001425, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7396, "confidence": 0.7396, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-probability-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 1562, "state_tokens_kept": 426, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029004959000001662, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "merchant_wins", "probabilities": {"customer_wins": 0.3836, "customer_withdraws": 0.1945, "merchant_wins": 0.422}, "confidence": 0.0442, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-probability-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 367, "state_tokens_kept": 367, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026828828999999388, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "bad_push", "probabilities": {"bad_push": 0.9114, "database_hardware": 0.0413, "upstream_provider": 0.0473}, "confidence": 0.6718, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 443, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-probability-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 919, "state_tokens_kept": 450, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027638167999999297, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5535, "confidence": 0.5535, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-probability-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 391, "state_tokens_kept": 391, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027936844999999266, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "no_show", "probabilities": {"attended": 0.2997, "late_cancellation": 0.096, "no_show": 0.6043}, "confidence": 0.1894, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 479, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-probability-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 820, "state_tokens_kept": 446, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028227060999999054, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "upheld", "probabilities": {"modified": 0.2461, "overturned": 0.1806, "upheld": 0.5733}, "confidence": 0.1142, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-tradeoff-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 667, "state_tokens_kept": 370, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02701731299999821, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "escalate_to_engineering", "probabilities": {"escalate_to_engineering": 0.2882, "escalate_to_security": 0.2579, "page_oncall": 0.1209, "process_refund": 0.2602, "send_kb_article": 0.0728}, "confidence": 0.0652, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-tradeoff-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 562, "state_tokens_kept": 415, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027611977000002952, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "order_b", "probabilities": {"order_a": 0.3254, "order_b": 0.3869, "order_c": 0.232, "order_d": 0.0556}, "confidence": 0.111, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-tradeoff-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 595, "state_tokens_kept": 432, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02704833399999984, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6102, "confidence": 0.6102, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-tradeoff-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 549, "state_tokens_kept": 375, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027204347999997935, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "p0_fix_24h", "probabilities": {"out_of_scope": 0.095, "p0_fix_24h": 0.3066, "p1_fix_7d": 0.2412, "p2_fix_30d": 0.2434, "p3_backlog": 0.1138}, "confidence": 0.0553, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-tradeoff-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 440, "state_tokens_kept": 429, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02702962400000075, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6266, "confidence": 0.6266, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-b-tradeoff-12", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 431, "state_tokens_kept": 400, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028288581999998286, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "refund_now", "probabilities": {"deny_refund": 0.1845, "escalate_to_fraud_team": 0.2514, "offer_store_credit": 0.0549, "refund_now": 0.3209, "wait_for_inspection": 0.1884}, "confidence": 0.0696, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 3101, "state_tokens_kept": 333, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.031133047000000857, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "erase_all", "probabilities": {"erase_all": 0.295, "erase_but_retain_transaction_records": 0.1939, "erase_marketing_only": 0.1409, "reject_unverified_request": 0.1155, "restrict_all_pending_legal_hold": 0.2548}, "confidence": 0.0357, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 3129, "state_tokens_kept": 401, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.0302771479999997, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "eur_150", "probabilities": {"eur_0": 0.2448, "eur_150": 0.2904, "eur_200": 0.2154, "eur_300": 0.1207, "eur_400": 0.0873, "eur_600": 0.0415}, "confidence": 0.0878, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2881, "state_tokens_kept": 430, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029683724000001632, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3051, "confidence": 0.6949, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2958, "state_tokens_kept": 420, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03030118800000281, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.0379, "legend": {"0": "Compliant: Named Users do not exceed Licences held on the Measurement Date", "1": "Shortfall of 1 to 5 Licences", "2": "Shortfall of 6 to 15 Licences", "3": "Shortfall of more than 15 Licences"}, "probabilities": {"0": 0.3764, "1": 0.3064, "2": 0.2201, "3": 0.0971}, "confidence": 0.0696, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2932, "state_tokens_kept": 410, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03121535900000083, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.508, "legend": {"0": "SEV-4 (lowest)", "1": "SEV-3", "2": "SEV-2", "3": "SEV-1 (highest)"}, "probabilities": {"0": 0.165, "1": 0.2694, "2": 0.458, "3": 0.1075}, "confidence": 0.0997, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 3145, "state_tokens_kept": 390, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03046758099999991, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "credit_minus_restocking_fee", "probabilities": {"credit_minus_restocking_fee": 0.7516, "full_credit": 0.0909, "reject_return": 0.0986, "repair_or_replace_only": 0.0589}, "confidence": 0.4029, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-long_policy-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 3127, "state_tokens_kept": 440, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02987892900000233, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7723, "confidence": 0.7723, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-probability-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 421, "state_tokens_kept": 412, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02642484000000067, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "halvorsen_polymer", "probabilities": {"halvorsen_polymer": 0.4348, "kuznets_technik": 0.2067, "ortega_seal_works": 0.3585}, "confidence": 0.039, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-temporal_numeric-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 685, "state_tokens_kept": 409, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026950142000000454, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6535, "confidence": 0.6535, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-temporal_numeric-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 631, "state_tokens_kept": 401, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026696637000000578, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "eur_688_00", "probabilities": {"eur_617_00": 0.1504, "eur_688_00": 0.3842, "eur_692_00": 0.1903, "eur_693_90": 0.1388, "eur_812_00": 0.1364}, "confidence": 0.0593, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-temporal_numeric-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 497, "state_tokens_kept": 378, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.0275396360000002, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "usd_2303_33", "probabilities": {"usd_2222_50": 0.2104, "usd_2303_01": 0.2428, "usd_2303_33": 0.2595, "usd_2335_00": 0.151, "usd_2340_00": 0.1362}, "confidence": 0.0191, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-temporal_numeric-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 710, "state_tokens_kept": 352, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027568515999998766, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "none_declined", "probabilities": {"none_declined": 0.4319, "sep_15_office_chair": 0.1611, "sep_16_taxi": 0.1221, "sep_17_parking": 0.0784, "sep_18_book_set": 0.07, "sep_21_client_dinner": 0.1366}, "confidence": 0.1232, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-temporal_numeric-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 684, "state_tokens_kept": 387, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02747705499999853, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "eur_498_81", "probabilities": {"eur_497_40": 0.245, "eur_498_81": 0.3132, "eur_506_06": 0.1572, "eur_513_69": 0.1382, "eur_532_01": 0.1464}, "confidence": 0.0346, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-opus-c-temporal_numeric-12", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 622, "state_tokens_kept": 412, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027485344999998773, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "expired_24_month_term", "probabilities": {"covered": 0.3034, "expired_24_month_term": 0.4103, "expired_30_month_cap": 0.2864}, "confidence": 0.0119, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-adversarial-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 210, "state_tokens_kept": 210, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02737284099999826, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "grant_access", "probabilities": {"deny_inactive_user": 0.1909, "deny_missing_approval": 0.2331, "grant_access": 0.3935, "grant_owner_exception": 0.1826}, "confidence": 0.0384, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 281, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-adversarial-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 172, "state_tokens_kept": 172, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02826184199999915, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8582, "confidence": 0.8582, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 221, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-adversarial-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 181, "state_tokens_kept": 181, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02745890400000306, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "retain_until_2031", "probabilities": {"delete_now": 0.2074, "needs_signature_review": 0.0894, "retain_indefinitely": 0.2951, "retain_until_2031": 0.4081}, "confidence": 0.0853, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 258, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-adversarial-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 165, "state_tokens_kept": 165, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02760583699999941, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7206, "confidence": 0.7206, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 207, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-adversarial-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 157, "state_tokens_kept": 157, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02769590900000196, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.6237, "legend": {"0": "Informational only.", "1": "Routine degradation.", "2": "Major multi-customer degradation.", "3": "Complete outage."}, "probabilities": {"0": 0.0403, "1": 0.4097, "2": 0.4361, "3": 0.1139}, "confidence": 0.2034, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 207, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-adversarial-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 199, "state_tokens_kept": 199, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.025273703999999952, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "standard_research", "probabilities": {"ethics_review": 0.2209, "major_grants": 0.3095, "manual_hold": 0.1161, "standard_research": 0.3536}, "confidence": 0.0521, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 272, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2274, "state_tokens_kept": 434, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02994450999999998, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "accept_s2_dusk", "probabilities": {"accept_s2_dusk": 0.8184, "reject_cutoff": 0.0546, "reject_tier": 0.0851, "reroute_s3": 0.0419}, "confidence": 0.5201, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2307, "state_tokens_kept": 442, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.030246387000001818, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "heron_q2", "probabilities": {"executive_bridge": 0.0996, "heron_q2": 0.5787, "insufficient_information": 0.1676, "security_primary": 0.1541}, "confidence": 0.1821, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2248, "state_tokens_kept": 441, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029871707999998165, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "pay_manager_approved", "probabilities": {"pay_manager_approved": 0.3879, "reject_category": 0.1178, "require_director": 0.3744, "require_finance_board": 0.1199}, "confidence": 0.1044, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2208, "state_tokens_kept": 443, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.030916552000000763, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "admit", "probabilities": {"admit": 0.534, "deny_prerequisite": 0.1267, "deny_score": 0.1907, "manual_equivalency": 0.1486}, "confidence": 0.1372, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2199, "state_tokens_kept": 439, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02983138699999799, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "local_maintenance", "probabilities": {"central_engineering": 0.1974, "insufficient_log": 0.267, "local_maintenance": 0.4527, "safety_shutdown": 0.0829}, "confidence": 0.1069, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2205, "state_tokens_kept": 435, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02961526300000017, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "reject_capacity", "probabilities": {"confirm_l3": 0.2853, "reject_capacity": 0.4051, "require_steward": 0.1281, "switch_l4": 0.1814}, "confidence": 0.0645, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2238, "state_tokens_kept": 437, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03005738299999905, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "needs_field_review", "probabilities": {"approve_45_days": 0.1741, "deny_sensitive": 0.1972, "needs_field_review": 0.4442, "reduce_to_30_days": 0.1845}, "confidence": 0.0646, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-multi_hop-12", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2201, "state_tokens_kept": 436, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028994967999999233, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "covered_recurrence", "probabilities": {"covered_recurrence": 0.4979, "expired": 0.128, "needs_authentication": 0.1697, "wrong_issue_class": 0.2045}, "confidence": 0.1085, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 225, "state_tokens_kept": 225, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026309466999997255, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8252, "confidence": 0.8252, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 273, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 133, "state_tokens_kept": 133, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02650790199999875, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8302, "confidence": 0.8302, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 176, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 241, "state_tokens_kept": 241, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027152677000000125, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "score", "score": 1.7646, "legend": {"0": "No operative exception.", "1": "Requested but unauthorized exception.", "2": "Authorized, unused exception.", "3": "Authorized exception already used."}, "probabilities": {"0": 0.1687, "1": 0.1717, "2": 0.3859, "3": 0.2737}, "confidence": 0.0443, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 295, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 231, "state_tokens_kept": 231, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027239227999999116, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "close_confirmed", "probabilities": {"await_internal_metrics": 0.1333, "close_agent_verified": 0.3405, "close_confirmed": 0.4464, "keep_open_unresolved": 0.0798}, "confidence": 0.1364, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 314, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 168, "state_tokens_kept": 168, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027465003999999738, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "on_time", "probabilities": {"canceled": 0.1081, "indeterminate": 0.0924, "late": 0.1781, "on_time": 0.6214}, "confidence": 0.2329, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 233, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 103, "state_tokens_kept": 103, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026902181000000525, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "ready_two_reviews", "probabilities": {"blocked_failed_test": 0.2364, "blocked_second_review": 0.168, "ready_one_review": 0.168, "ready_two_reviews": 0.4275}, "confidence": 0.0596, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 183, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-13", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 131, "state_tokens_kept": 131, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02606134099999835, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7716, "confidence": 0.7716, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 179, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-a-trap-15", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 217, "state_tokens_kept": 217, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02717121699999936, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8548, "confidence": 0.8548, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 267, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 158, "state_tokens_kept": 158, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026706145999998654, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3817, "confidence": 0.6183, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 218, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 113, "state_tokens_kept": 113, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027440813999998426, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.7656, "confidence": 0.7656, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 173, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 160, "state_tokens_kept": 160, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027295139999999662, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6026, "confidence": 0.6026, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 220, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 99, "state_tokens_kept": 99, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02670822599999667, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.2228, "confidence": 0.7772, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 159, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 122, "state_tokens_kept": 122, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02662734400000133, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6631, "confidence": 0.6631, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 182, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-14", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 154, "state_tokens_kept": 154, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02674904799999922, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6352, "confidence": 0.6352, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 214, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-15", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 103, "state_tokens_kept": 103, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026681146000001377, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5, "confidence": 0.5, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 163, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-judge_hard-18", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 89, "state_tokens_kept": 89, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026497351999999808, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8957, "confidence": 0.8957, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 149, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-long_policy-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2610, "state_tokens_kept": 413, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03108123599999857, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "approve_all", "probabilities": {"approve_all": 0.3648, "approve_lodging_only": 0.3028, "approve_meals_only": 0.1811, "reject_entire_claim": 0.1513}, "confidence": 0.0444, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-long_policy-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2498, "state_tokens_kept": 417, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.030881732000000994, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "privacy_and_security", "probabilities": {"analytics_owner_only": 0.3064, "deny_current_export": 0.1722, "privacy_and_security": 0.3563, "privacy_only": 0.1652}, "confidence": 0.0403, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-long_policy-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2436, "state_tokens_kept": 407, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03042068099999895, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "delete_profile_omar_april_and_table", "probabilities": {"delete_everything": 0.235, "delete_nothing": 0.0931, "delete_only_profile": 0.2478, "delete_profile_omar_april_and_table": 0.424}, "confidence": 0.0832, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-long_policy-06", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2468, "state_tokens_kept": 423, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.030890391999999878, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "harbor_neighbors", "probabilities": {"east_ward": 0.0102, "harbor_neighbors": 0.9526, "no_award": 0.0302, "river_arts": 0.0071}, "confidence": 0.8315, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-routing_hard-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 181, "state_tokens_kept": 181, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028388414999998446, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "spreadsheet_formula", "probabilities": {"data_pipeline": 0.1616, "desktop_support": 0.2193, "finance_review": 0.1288, "office_file_repair": 0.2278, "spreadsheet_formula": 0.2625}, "confidence": 0.0187, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 316, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-routing_hard-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 171, "state_tokens_kept": 171, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02803482699999904, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "payment_disputes", "probabilities": {"customer_support": 0.1914, "fraud_review": 0.0721, "ledger_reconciliation": 0.1814, "mobile_app": 0.2129, "payment_disputes": 0.3423}, "confidence": 0.0606, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 286, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-routing_hard-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 182, "state_tokens_kept": 182, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028297642999998374, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "api_platform", "probabilities": {"api_platform": 0.404, "authoritative_dns": 0.22, "endpoint_devices": 0.0719, "network_dns": 0.2175, "tls_certificates": 0.0866}, "confidence": 0.1101, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 301, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-routing_hard-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 156, "state_tokens_kept": 156, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02785516299999813, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "release_notarization", "probabilities": {"ci_infrastructure": 0.2834, "code_signing": 0.1876, "endpoint_security": 0.052, "macos_runtime": 0.1631, "release_notarization": 0.3139}, "confidence": 0.0776, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 266, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-routing_hard-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 162, "state_tokens_kept": 162, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026651704999999026, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "repository_admin", "probabilities": {"git_client": 0.168, "identity_login": 0.21, "repository_admin": 0.2464, "source_access": 0.1822, "vpn_network": 0.1934}, "confidence": 0.0055, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 278, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-temporal_numeric-01", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 133, "state_tokens_kept": 133, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027086666000002424, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "within_window", "probabilities": {"before_open": 0.2501, "late_by_2h_or_more": 0.2108, "late_by_under_2h": 0.1661, "within_window": 0.3729}, "confidence": 0.0328, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 232, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-temporal_numeric-02", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 136, "state_tokens_kept": 136, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028021247000001637, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "at_least_100gb_below", "probabilities": {"at_least_100gb_below": 0.3052, "exactly_equal": 0.259, "over_limit": 0.2522, "under_100gb_below": 0.1836}, "confidence": 0.0112, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 231, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-temporal_numeric-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 101, "state_tokens_kept": 101, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026292757000000222, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "12_credits", "probabilities": {"12_credits": 0.3571, "15_credits": 0.1829, "6_credits": 0.2108, "9_credits": 0.2491}, "confidence": 0.0241, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 184, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-b-temporal_numeric-04", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 133, "state_tokens_kept": 133, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027419242000000565, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "pass", "probabilities": {"clear_fail": 0.2185, "narrow_fail": 0.3179, "pass": 0.4636}, "confidence": 0.0415, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 196, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-03", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 331, "state_tokens_kept": 331, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027338590999999468, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8035, "confidence": 0.8035, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 402, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-05", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 238, "state_tokens_kept": 238, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027306980999998842, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.344, "confidence": 0.656, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 309, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 211, "state_tokens_kept": 211, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.02861565999999982, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3533, "confidence": 0.6467, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 282, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-08", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 225, "state_tokens_kept": 225, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026013480999999672, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.6125, "confidence": 0.6125, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 296, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 219, "state_tokens_kept": 219, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026283267000000166, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.3942, "confidence": 0.6058, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 290, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-10", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 236, "state_tokens_kept": 236, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027621377999999197, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.5836, "confidence": 0.5836, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 307, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 255, "state_tokens_kept": 255, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.028278461999999394, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8679, "confidence": 0.8679, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 326, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-13", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 214, "state_tokens_kept": 214, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.027294200000000046, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.1605, "confidence": 0.8395, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 285, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-judge_hard-15", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 199, "state_tokens_kept": 199, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026648995999998704, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "noul", "noul": 0.8325, "confidence": 0.8325, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 270, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-multi_hop-07", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2379, "state_tokens_kept": 430, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03020296600000094, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "programme_review", "probabilities": {"partial_release": 0.1026, "programme_review": 0.3709, "reject_draw": 0.1654, "release_draw": 0.3611}, "confidence": 0.0861, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-multi_hop-09", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2377, "state_tokens_kept": 423, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.03193643499999865, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "terminate_oakmere", "probabilities": {"bus_substitution": 0.0812, "call_fen_cross": 0.088, "pass_fen_cross": 0.0737, "terminate_oakmere": 0.7572}, "confidence": 0.4082, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-multi_hop-11", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2366, "state_tokens_kept": 424, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.029608452999998036, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "coverage_confirmed", "probabilities": {"adjuster_referral": 0.1365, "coverage_confirmed": 0.3651, "coverage_excluded": 0.2902, "documentation_needed": 0.2081}, "confidence": 0.0439, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-multi_hop-12", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 402, "state_tokens_kept": 402, "state_truncated": false, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.026940931999998696, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "supervised_delivery", "probabilities": {"conservation_hold": 0.1809, "deny_request": 0.0748, "digital_copy_only": 0.2204, "supervised_delivery": 0.5238}, "confidence": 0.1522, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 489, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}
{"id": "benchmark:hard-sol-c-multi_hop-13", "runtime": {"model": "Laya English base", "checkpoint_revision": "1c5edc17a7acd8701df6fc341c0d179f1c62c982", "source_commit": "42626c348753fbb17572a813127df2278a1ec527", "gpu": "NVIDIA L40S", "python": "3.12.10", "software": {"torch": "2.8.0", "transformers": "5.0.0", "numpy": "2.5.3", "safetensors": "0.6.2"}, "checkpoint_sha256": "891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c", "weights_dtype": "torch.float32", "autocast_dtype": "torch.bfloat16", "max_length": 512, "head_max_length": 192, "temperature": [1.6369030475616455, 1.2514300346374512, 1.983399510383606], "temperature_by_options": {"choice:3-5": 1.7601518630981445, "choice:6-10": 1.0000158548355103, "score:3-5": 1.2514300346374512, "noul:2": 1.983399510383606, "choice:11+": 0.10058280825614929, "choice:2": 1.9063563346862793}, "probability_precision": "native API,4 decimal places", "prefix_cache": false}, "input_audit": {"state_tokens_before": 2445, "state_tokens_kept": 433, "state_truncated": true, "instruction_truncated": false, "options_truncated": false}, "server_wall_seconds": 0.0294356889999996, "ok": true, "response": {"model": "laya-rl-agent", "answers": {"decision": {"type": "choice", "choice": "normal_release", "probabilities": {"engineering_review": 0.2409, "normal_release": 0.5278, "rework": 0.1809, "scrap_lot": 0.0504}, "confidence": 0.1776, "action": {"act_probability": 1.0}}}, "usage": {"input_tokens": 512, "output_tokens": 0}}, "gpu_peak_allocated_bytes": 2449583104}